Initial commit

.gitignore

+log.*
+/data/
+/pictures
+*.png
+*.dot
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

README.md

+# Build the population tree from D statistics.
+
+## Prepare
+
+python==3.6.8
+
+Install graphviz using apt-get or conda.
+```
+conda install graphviz
+```
+Install python packages.
+```
+pip install -r requirements.txt
+``` 
+
+## Run
+
+Configure data file, save path, threshold, and populations.
+
+```
+python main.py
+```

geneflow_P.py

+
+# D1 = d of D1(S2, Tar; S1, Og)[4]
+# Z1 = Z of D1(S2, Tar; S1, Og)[5]
+# D2 = d of D2(S2, S1'; S1, Og)[4]
+# Z2 = Z of D2(S2, S1'; S1, Og)[5]
+# Mix = mixture from S1 to Tar
+# M_PD = the Probability distribution of the mixture
+
+def Probability_distribution(D1,D2,Z1,Z2):
+    import matplotlib.pyplot as plt
+    import numpy as np
+    #x=D2(S1'):
+    u2 = D2
+    se2 = D2/Z2 
+    #y=D1(Tar):
+    u1 = D1
+    se1 = D1/Z1
+    Mix=D1/D2
+    import scipy.integrate
+    from numpy import exp
+    from scipy.stats import norm
+    if Mix >=0 and Mix <=1:
+        f = lambda y,x: (norm.pdf(x, loc=u2, scale=se2))*(norm.pdf(y, loc=u1, scale=se1))
+        g = 0
+        h = 1
+        i1_1 = scipy.integrate.dblquad(f, 0, 1, g, h)[0]
+        g = 0
+        h = 1
+        i1_2 = scipy.integrate.dblquad(f, -1, 0, g, h)[0]
+        g = -1
+        h = 0
+        i1_3 = scipy.integrate.dblquad(f, -1, 0, g, h)[0]
+        g = -1
+        h = 0
+        i1_4 = scipy.integrate.dblquad(f, 0, 1, g, h)[0]
+        #i1_1-i1_4:1st-4th quadrant
+        g = -1
+        h = 1
+        i1_5 = scipy.integrate.dblquad(f, -1, 1, g, h)[0]
+        #i1_5:x[-1:1],y[-1,1]
+        g = lambda x: x
+        h = 0
+        i2 = scipy.integrate.dblquad(f, -1, 0, g, h)[0]
+        g = 0
+        h = lambda x: x
+        i3 = scipy.integrate.dblquad(f, 0, 1, g, h)[0]
+        i=i2+i3
+        if i1_5 <= 0 or i1_5 <= i:
+            i1=i1_1 + i1_2 + i1_3 + i1_4
+        if i1_5 > 0 and i1_5 >= i:
+            i1=i1_5
+        M_PD=i/i1
+        if M_PD > 1:
+            M_PD=1
+    else:
+       M_PD="Warning: No calculation required!" 
+    return(Mix,M_PD)

preprocess.py

+DEBUG = False
+triangle_templates = [
+#    [ 0, 0, 0],
+#    [ 1,-1, 1],
+#    [ 1,-1, 0],
+#    [ 1,-1,-1],
+#    [-1,-1, 1],
+#    [-1, 0, 1],
+#    [-1, 1, 1],
+#    [ 1, 1,-1],
+#    [ 0, 1,-1],
+#    [-1, 1,-1],
+#    [ 1, 0,-1],
+#    [-1, 1, 0],
+#    [ 0,-1, 1],
+
+#    [ 0, 0, 0],
+
+#    [ 10,-10, 10],
+#    [ 10,-10, 0],
+#    [ 10,-10,-10],
+#    [-10,-10, 10],
+#    [-10, 0,  10],
+#    [-10, 10, 10],
+#    [ 10, 10,-10],
+#    [ 0,  10,-10],
+#    [-10, 10,-10],
+
+    [ 10, 10,-10],
+    [ 10, 0 ,-10],
+    [ 10,-10,-10],
+    [-10, 10,-10],
+    [-10, 10, 0 ],
+    [-10, 10, 10],
+    [ 10,-10, 10],
+    [ 0 ,-10, 10],
+    [-10,-10, 10],
+]
+redstar_index = [2, 2, 2, 1, 1, 1, 0, 0, 0]
+
+def adjust_D_bak(triangle, threshold):
+    adjusted_data = []
+    to_adjust = []
+    for i, value in enumerate(triangle):
+        if abs(value) < threshold:
+            adjusted_data.append(0)
+            to_adjust.append((i, value))
+        elif value >= threshold:
+            adjusted_data.append(10)
+        elif value <= -threshold:
+            adjusted_data.append(-10)
+    to_adjust = sorted(to_adjust, key=lambda x: abs(x[1]), reverse=True)
+    adjusted_value = []
+    adjust_threshold = None 
+    while adjusted_data not in triangle_templates:
+        if len(to_adjust) == 0:
+            raise Exception('Cannot adjust triangle: %s' % triangle)
+        adjust_idx, adjust_threshold = to_adjust.pop(0)
+        adjusted_data[adjust_idx] = 10 if adjust_threshold > 0 else -10
+        adjusted_value.append((adjust_idx, adjust_threshold))
+    return adjusted_value, adjusted_data 
+
+def adjust_D(key, triangle, threshold, min_redstar, s1_record):
+    adjusted_data = []
+    to_adjust = []
+    for i, value in enumerate(triangle):
+#        if abs(value) < threshold:
+#            adjusted_data.append(0)
+#            to_adjust.append((i, value))
+        if value > 0:
+            adjusted_data.append(10)
+        elif value < 0:
+            adjusted_data.append(-10)
+        else:
+            raise ValueError('value is 0')
+    if adjusted_data not in triangle_templates:
+        raise Exception('Cannot adjust triangle: %s, %s' % (key, triangle))
+    index = triangle_templates.index(adjusted_data)
+    redstar_idx = redstar_index[index]
+
+
+    def add_s1_record(s2, tar, s1):
+        if (key[s2], key[tar]) not in s1_record:
+            s1_record[(key[s2], key[tar])] = []
+        s1_record[(key[s2], key[tar])].append(key[s1])
+    T = 0
+    A = 1
+    B = 2
+    if redstar_idx == 2:
+        add_s1_record(T, A, B)
+        add_s1_record(B, A, T)
+    elif redstar_idx == 1:
+        add_s1_record(T, B, A)
+        add_s1_record(A, B, T)
+    elif redstar_idx == 0:
+        add_s1_record(A, T, B)
+        add_s1_record(B, T, A)
+    if DEBUG:
+        print('------------------------------')
+        print(key)
+        print(triangle)
+
+    min_redstar.append(abs(triangle[redstar_idx]))
+    if abs(triangle[redstar_idx]) < threshold:
+        adjusted_data[redstar_idx] = 0
+    return None, adjusted_data, redstar_idx 
+
+def get_adjusted_D_statistics(D_statistics, adjusted_D_values, triangles, A, B, C, f4, value, threshold, min_redstar, redstar_indices, key_visited, s1_record):
+    key = tuple(sorted([A, B, C]))
+    if key in key_visited: return
+    pos = key.index(C)
+    sign = 1 if (key.index(A) - pos) % 3 < (key.index(B) - pos) % 3 else -1
+    value *= sign
+    f4 *= sign
+    # print(key, value)
+    if key not in triangles:
+        triangles[key] = {'D':[None, None, None], 'f4':[None, None, None]}
+    triangles[key]['f4'][pos] = f4
+    triangles[key]['D'][pos] = value
+
+    if None not in triangles[key]['D']:
+        key_visited.append(key)
+        adjusted_value, adjusted_data, redstar_index = adjust_D(key, triangles[key]['D'], threshold, min_redstar, s1_record)
+        redstar_indices[key] = redstar_index
+#        if len(adjusted_value) > 0:
+#            for idx, value in adjusted_value:
+#                adjusted_D_values.append((key[idx], key[(idx+1)%3], key[(idx+2)%3], value))
+        for i, value in enumerate(adjusted_data):
+            # value == 0: record the original number
+            if value == 0:
+                D_statistics.append((key[(i+1)%3], key[(i+2)%3], key[i], value, triangles[key]['D'][i]))
+            else:
+                D_statistics.append((key[(i+1)%3], key[(i+2)%3], key[i], value, triangles[key]['D'][i]))
+        print_key = key[:] + key[:1]
+        # print(print_key, triangles[key], adjusted_data)
+#        if key == ('Dushan', 'Longlin', 'Qihe'):
+#            print(D_statistics)
+        # del triangles[key]
\ No newline at end of file

propotion.py

+import matplotlib.pyplot as plt
+from sklearn.linear_model import LinearRegression
+import numpy as np
+
+
+
+populations = ['Kostenki14',
+               'Tianyuan',
+               'Boshan',
+               'Qihe',
+               'Longlin',
+               'Dushan',
+               'Baojianshan',
+               'La368']
+
+all_statistics = dict()
+all_populations = set()
+with open('data/UPA34_4.f4.AI.log') as f:
+    for l in f.readlines():
+        if not l.startswith('result'): continue
+        A, B, C = l.split()[1:4]
+        all_populations.add(A)
+        all_populations.add(B)
+        all_populations.add(C)
+        value = float(l.split()[5])
+        all_statistics[(A, B, C)] = value
+
+test = 'Dushan'
+R1 = 'Baojianshan'
+x = []
+y = [] 
+print(all_populations)
+for R2 in all_populations:
+    if R2 != 'Qihe': continue
+    for O1 in all_populations:
+        if O1 in populations: continue
+        if R2 == O1 or R1 == test or R2 == test or R1 == O1: continue
+        y.append(all_statistics[(test, R1, O1)])
+        x.append(all_statistics[(test, R2, O1)])
+    
+print(len(x))
+
+plt.scatter(x, y)
+plt.savefig('test.png')
+
+x = np.array(x).reshape(-1, 1)
+y = np.array(y).reshape(-1, 1)
+model = LinearRegression()
+model.fit(x, y)
+
+print("k", model.coef_)
+print("b", model.intercept_)
+print("R2", model.score(x, y))
\ No newline at end of file

requirements.txt

+anytree==2.8.0
+cvxopt==1.3.0
+graphviz==0.19.1
+matplotlib==3.3.4
+networkx==2.5.1
+numpy==1.19.5
+Pillow==8.4.0
+pydot==1.4.2
+pyparsing==3.0.9
+scikit-learn==0.24.2
+six==1.16.0

test_cases.py

+
+class TestCase(object):
+    def __init__(self, populations=['A', 'B', 'T'], D_key=[('A', 'B', 'T'), ('A', 'T', 'B'), ('B', 'T', 'A')], D_value=[0, 0, 0]):
+        self.populations = populations
+        self.D_key = D_key
+        self.D_value = D_value
+        self.D_statistics = []
+        for k, v in zip(self.D_key, self.D_value):
+            self.D_statistics.append((*k, v))
+
+D_key = [('A', 'B', 'T'), ('A', 'T', 'B'), ('B', 'T', 'A')]
+populations = ['A', 'B', 'T']
+three_population_cases = []
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [ 0, 0, 0]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [ 1, 1, 1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [ 1, 1, 0]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [ 1, 1,-1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [-1, 1, 1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [-1, 0, 1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [-1,-1, 1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [ 1,-1,-1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [ 0,-1,-1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [-1,-1,-1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [ 1, 0,-1]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [-1,-1, 0]))
+three_population_cases.append(TestCase(['A', 'B', 'T'], D_key, [ 0, 1, 1]))
\ No newline at end of file

utils.py

+from preprocess import get_adjusted_D_statistics
+from itertools import combinations
+import copy
+
+def read_data(data_path, threshold, populations):
+    D_statistics = []
+    adjusted_D_values = []
+    triangles = {}
+    redstar_indices = {}
+    key_visited = []
+    eq_scc = None
+    min_redstar = []
+    s1_record = {}
+    with open(data_path) as f:
+        for l in f.readlines():
+            if not l.startswith('result'): continue
+            A, B, C = l.split()[1:4]
+            if A not in populations or B not in populations or C not in populations: continue
+            f4 = float(l.split()[5])
+            value = float(l.split()[6])
+            get_adjusted_D_statistics(D_statistics, adjusted_D_values, triangles, A, B, C, f4, value, threshold, min_redstar, redstar_indices, key_visited, s1_record)
+            if len(D_statistics) >= len(populations) * (len(populations) - 1) * (len(populations) - 2) / 2:
+                break
+    return D_statistics, adjusted_D_values, triangles, redstar_indices, min_redstar, s1_record
+
+def equal_vertices_judgement(g, vertices):
+    edges = list(combinations(vertices, 2))
+    for v1, v2 in edges:
+#        print(v1, v2)
+#        print(g.graph[v1]['eq'])
+#        print(g.graph[v2]['eq'])
+        if v2 in [eq[0] for eq in g.graph[v1]['eq']] or v1 in [eq[0] for eq in g.graph[v2]['eq']]: continue
+        if v2 in g.graph[v1]['out'] or v2 in g.graph[v1]['in']: return False
+    return True
+
+
+from collections import defaultdict
+
+class Graph:
+    def __init__(self, vertices):
+        self.V = vertices
+        self.adj = defaultdict(list)
+
+    def addEdge(self, u, v):
+        self.adj[u].append(v)
+
+    def DFS(self, v, visited, stack):
+        visited[v] = True
+        stack.append(v)
+
+        for neighbor in self.adj[v]:
+            if not visited[neighbor]:
+                if self.DFS(neighbor, visited, stack):
+                    return True
+            elif neighbor in stack:
+                cycle = []
+                index = stack.index(neighbor)
+                for i in range(index, len(stack)):
+                    cycle.append(stack[i])
+                return cycle
+
+        stack.pop()
+        return False
+
+    def findCycles(self):
+        visited = [False] * self.V
+        stack = []
+        for i in range(self.V):
+            if not visited[i]:
+                result = self.DFS(i, visited, stack)
+                if result != False:
+                    return result
+
+import networkx as nx 
+
+def simple_cycles(G, eq_edges=[], edges_to_adjust=[]):
+    def _unblock(thisnode, blocked, B):
+        stack = {thisnode}
+        while stack:
+            node = stack.pop()
+            if node in blocked:
+                blocked.remove(node)
+                stack.update(B[node])
+                B[node].clear()
+
+    # Johnson's algorithm requires some ordering of the nodes.
+    # We assign the arbitrary ordering given by the strongly connected comps
+    # There is no need to track the ordering as each node removed as processed.
+    # Also we save the actual graph so we can mutate it. We only take the
+    # edges because we do not want to copy edge and node attributes here.
+    subG = type(G)(G.edges())
+    sccs = [scc for scc in nx.strongly_connected_components(subG) if len(scc) > 2]
+
+    # Johnson's algorithm exclude self cycle edges like (v, v)
+    # To be backward compatible, we record those cycles in advance
+    # and then remove from subG
+    for v in subG:
+        if subG.has_edge(v, v):
+            # yield [v]
+            subG.remove_edge(v, v)
+    
+    cycle_ids = set()
+
+    visited_startnodes = set()
+    cycles = []
+    while sccs:
+        scc = sccs.pop()
+        sccG = subG.subgraph(scc)
+        # order of scc determines ordering of nodes
+        # print('scc', scc)
+        for startnode in scc:
+            if startnode in visited_startnodes:
+                continue
+            visited_startnodes.add(startnode)
+            # startnode = scc.pop()
+
+            # Processing node runs "circuit" routine from recursive version
+            path = [startnode]
+            blocked = set()  # vertex: blocked from search?
+            closed = set()  # nodes involved in a cycle
+            blocked.add(startnode)
+            B = defaultdict(set)  # graph portions that yield no elementary circuit
+
+            startnbrs = list(sccG[startnode])
+            for node in copy.deepcopy(startnbrs):
+                if (startnode, node) in eq_edges:
+                    startnbrs.remove(node)
+            if not startnbrs:
+                continue
+            
+            # print(startnode, startnbrs)
+            stack = [(startnode, copy.deepcopy(startnbrs))]  # sccG gives comp nbrs
+
+            while stack:
+                thisnode, nbrs = stack[-1]
+                if nbrs:
+                    nextnode = nbrs.pop()
+                    #print('nextnode', nextnode)
+                    if nextnode == startnode:
+                        # paths.add(''.join(path[:]))
+                        # print(path)
+                        edge_id = [0] * len(edges_to_adjust)
+                        for node_idx in range(len(path)):
+                            edge = (path[node_idx], path[(node_idx + 1) % len(path)])
+                            if edge in edges_to_adjust:
+                                edge_id[edges_to_adjust.index(edge)] = 1
+                        cycle_ids.add(tuple(edge_id))
+                        #yield path[:]
+                        closed.update(path)
+                        # print('path', path)
+                        #print('closed', closed)
+                    #                        #print "Found a cycle", path, closed
+                    elif nextnode not in blocked:
+                        path.append(nextnode)
+                        stack.append((nextnode, list(sccG[nextnode])))
+                        closed.discard(nextnode)
+                        blocked.add(nextnode)
+                        #print('path', path)
+                        continue
+                # done with nextnode... look for more neighbors
+                if not nbrs:  # no more nbrs
+                    if thisnode in closed:
+                        _unblock(thisnode, blocked, B)
+                    else:
+                        for nbr in sccG[thisnode]:
+                            if thisnode not in B[nbr]:
+                                B[nbr].add(thisnode)
+                    stack.pop()
+                    #                assert path[-1] == thisnode
+                    path.pop()
+            # done processing this node
+            # print('paths', paths)
+            H = subG.subgraph(scc)  # make smaller to avoid work in SCC routine
+            H = type(G)(H.edges())
+            # print('remove', startnode, startnbrs)
+            # for node in startnbrs:
+            for node in startnbrs:
+                # print('remove', startnode, node)
+                H.remove_edge(startnode, node) 
+                subG.remove_edge(startnode, node) 
+            # for new_scc in nx.strongly_connected_components(H):
+                # print('new_scc', new_scc)
+            sccs.extend(new_scc for new_scc in nx.strongly_connected_components(H) if len(new_scc) > 2)
+            del H
+            break
+            #print(startnode, 'finished')
+    
+    return cycle_ids 
+
+
+if __name__ == '__main__':
+#    g = Graph(4)
+#    g.addEdge(0, 1)
+#    g.addEdge(0, 2)
+#    g.addEdge(1, 2)
+#    g.addEdge(2, 0)
+#    g.addEdge(2, 3)
+#    g.addEdge(3, 3)
+#
+#    print(g.findCycles())
+
+    import networkx as nx
+
+    # Create Directed Graph
+    G=nx.DiGraph()
+
+    # Add a list of nodes:
+    G.add_nodes_from(["a","b","c","d","e"])
+
+    # Add a list of edges:
+    # G.add_edges_from([("a","b"),("b","c"), ("c","a"), ("b","d"), ("d","e"), ("e","a")])
+    G.add_edges_from([('b','a'), ('c','b'), ("a","b"),("b","c"), ("c","a"), ("b","d"), ("d","a")])
+    # G.add_edges_from([('a','d'), ('a','c'), ('b','a'), ('c','b'), ("a","b"),("b","c"), ("c","a")])
+
+    #Return a list of cycles described as a list o nodes
+    print(list(simple_cycles(G, [('a', 'b'), ('b', 'c'), ('b', 'a'), ('c', 'b')], [('c', 'a')])))
+
+#    G=nx.DiGraph()
+#
+#    # Add a list of nodes:
+#    G.add_nodes_from([1,2,3,4,5])
+#
+#    # Add a list of edges:
+#    G.add_edges_from([(1,2), (2,1)])
+
+    #Return a list of cycles described as a list o nodes
+#    cycles = list(nx.simple_cycles(G))
+#    print(cycles)
+#    new_cycles = []
+#    for cycle in cycles:
+#        edges = []
+#        for i in range(len(cycle)):
+#            edges.append((cycle[i], cycle[(i+1) % len(cycle)]))
+#        new_cycles.append(edges)
+#    cycles = new_cycles
+#
+#    import cvxopt
+#    from cvxopt import matrix
+#    from cvxopt.glpk import ilp
+#    import numpy as np
+#
+#    num_edges = 200
+#    num_cycles = 100
+#    edges = list(G.edges)
+#    c = matrix(np.ones(num_edges, dtype=float))
+#    G2 = matrix(np.zeros((num_cycles, num_edges), dtype=float))
+#    print(cycles)
+#    print(edges[0])
+#    for i in range(num_cycles):
+#        for j in range(num_edges):
+#            if edges[j % 6] in cycles[i % 2]:
+#                G2[i,j] = -1
+#    h = matrix(-1 * np.ones(num_cycles, dtype=float))
+#    I = set()
+#    B = set(range(num_edges))
+#    (status,x)=ilp(c,G2,h,B=B)
+#    print(status)
+#    print(x)
+#
\ No newline at end of file