diff --git a/project/Build.scala b/project/Build.scala index b2423aa8..f45cb880 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -10,7 +10,7 @@ object ApplicationBuild extends Build { baseDirectory in run := file("."), baseDirectory in Test := file("."), - scalaVersion := "2.11.7", + scalaVersion := "2.12.4", scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature"), showSuccess := false, @@ -24,8 +24,8 @@ object ApplicationBuild extends Build { unmanagedBase <<= baseDirectory { base => base / "lib" }, libraryDependencies ++= Seq( - "org.scalatest" % "scalatest_2.11" % "2.2.4" % "test", - "com.github.scopt" %% "scopt" % "3.3.0" + "org.scalatest" % "scalatest_2.12" % "3.0.1" % "test", + "com.github.scopt" %% "scopt" % "3.7.0" ), mainClass in (Compile, run) := Some("tps.Main"), diff --git a/scripts/compare-undirected-networks b/scripts/compare-undirected-networks new file mode 100755 index 00000000..5863e8e2 --- /dev/null +++ b/scripts/compare-undirected-networks @@ -0,0 +1,8 @@ +#!/bin/bash + +NETWORK_1=$1 +NETWORK_2=$2 + +sbt \ + "project tps-core" \ + "run-main tps.evaluation.UndirectedNetworkComparison $NETWORK_1 $NETWORK_2" diff --git a/scripts/generate-scores-and-prizes b/scripts/generate-scores-and-prizes new file mode 100755 index 00000000..5b044eae --- /dev/null +++ b/scripts/generate-scores-and-prizes @@ -0,0 +1,9 @@ +#!/bin/bash + +TIME_SERIES_FILE=$1 +MAPPING_FILE=$2 +FOLD_CHANGE_THRESHOLD=$3 + +sbt \ + "project tps-core" \ + "run-main tps.evaluation.SignificanceScoreAndPrizeProducer $TIME_SERIES_FILE $MAPPING_FILE $FOLD_CHANGE_THRESHOLD" diff --git a/scripts/permute-time-series-and-scores b/scripts/permute-time-series-and-scores new file mode 100755 index 00000000..aeaa9f10 --- /dev/null +++ b/scripts/permute-time-series-and-scores @@ -0,0 +1,13 @@ +#!/bin/bash + +TIMESERIES=data/timeseries/median-time-series.tsv +FIRSTSCORES=data/timeseries/p-values-first.tsv +PREVSCORES=data/timeseries/p-values-prev.tsv + +for i in `seq 1 100`; +do + OUTFOLDER="permutation-$i" + sbt \ + "project tps-core" \ + "run-main tps.evaluation.TimeSeriesPermutation $TIMESERIES $FIRSTSCORES $PREVSCORES $i $OUTFOLDER" +done diff --git a/scripts/run-funchisq-to-sif b/scripts/run-funchisq-to-sif new file mode 100755 index 00000000..c506cb08 --- /dev/null +++ b/scripts/run-funchisq-to-sif @@ -0,0 +1,10 @@ +#!/bin/bash + +NETWORK_FILE=$1 +MAPPING_FILE=$2 +P_VAL_THRESHOLD=$3 +MAX_NB_EDGES=$4 + +sbt \ + "project tps-core" \ + "run-main tps.evaluation.funchisq.FunChisq2SIFConverter $NETWORK_FILE $MAPPING_FILE $P_VAL_THRESHOLD $MAX_NB_EDGES" diff --git a/scripts/run-network-analysis b/scripts/run-network-analysis new file mode 100755 index 00000000..76027861 --- /dev/null +++ b/scripts/run-network-analysis @@ -0,0 +1,9 @@ +#!/bin/bash + +for reference in data/resources/reference/*.tsv +do + echo "Evaluating against $reference" + sbt \ + "project tps-core" \ + "run-main tps.evaluation.NetworkReferenceComparison $reference $*" +done diff --git a/scripts/run-one-to-many-network-comparison b/scripts/run-one-to-many-network-comparison new file mode 100755 index 00000000..a80602e5 --- /dev/null +++ b/scripts/run-one-to-many-network-comparison @@ -0,0 +1,5 @@ +#!/bin/bash + +sbt \ + "project tps-core" \ + "run-main tps.evaluation.DirectedNetworkComparison $*" diff --git a/scripts/run-overlap-analysis b/scripts/run-overlap-analysis new file mode 100755 index 00000000..c0dfaf07 --- /dev/null +++ b/scripts/run-overlap-analysis @@ -0,0 +1,13 @@ +#!/bin/bash + +timeseries=data/timeseries/median-time-series.tsv +firstscores=data/timeseries/p-values-first.tsv +prevscores=data/timeseries/p-values-prev.tsv +mapping=data/timeseries/peptide-mapping.tsv + +for reference in data/resources/reference/*.tsv +do + sbt \ + "project tps-core" \ + "run-main tps.evaluation.OverlapAnalysis $reference $timeseries $mapping $firstscores $prevscores" +done diff --git a/scripts/run-permuted-time-series b/scripts/run-permuted-time-series new file mode 100755 index 00000000..53d4a41a --- /dev/null +++ b/scripts/run-permuted-time-series @@ -0,0 +1,15 @@ +#!/bin/bash + +for i in `seq 1 100`; +do + ./scripts/run \ + --network data/networks/input-network.tsv \ + --timeseries data/timeseries/permuted-time-series/permutation-$i/permuted-time-series.tsv \ + --firstscores data/timeseries/permuted-time-series/permutation-$i/permuted-first-scores.tsv \ + --prevscores data/timeseries/permuted-time-series/permutation-$i/permuted-prev-scores.tsv \ + --partialmodel data/resources/kinase-substrate-interactions.sif \ + --peptidemap data/timeseries/peptide-mapping.tsv \ + --source EGF_HUMAN \ + --threshold 0.01 \ + --outfolder permuted-time-series-run-$i +done diff --git a/scripts/run-txn b/scripts/run-txn new file mode 100755 index 00000000..aa6e33dc --- /dev/null +++ b/scripts/run-txn @@ -0,0 +1,68 @@ +#!/bin/bash + +NETWORKFOLDER=data/networks/evaluation +KINSUB=data/resources/kinase-substrate-interactions.sif + +TXN_ALL_UNDIRECTED=$NETWORKFOLDER/TXN-ALL-undirected.tsv +TXN_ALL_DIRECTED=$NETWORKFOLDER/TXN-ALL-directed.sif +TXN_1F_UNDIRECTED=$NETWORKFOLDER/TXN-1F-undirected.tsv +TXN_1F_DIRECTED=$NETWORKFOLDER/TXN-1F-directed.sif + +function run_with_base_args() { + ./scripts/run \ + --timeseries data/timeseries/median-time-series.tsv \ + --firstscores data/timeseries/p-values-first.tsv \ + --prevscores data/timeseries/p-values-prev.tsv \ + --peptidemap data/timeseries/peptide-mapping.tsv \ + --source EGFR_HUMAN \ + --threshold 0.01 \ + "$@" +} + +run_with_base_args \ + --outlabel TXN-ALL-directed-topo-kinsub \ + --network $TXN_ALL_UNDIRECTED \ + --partialmodel $KINSUB \ + --partialmodel $TXN_ALL_DIRECTED \ + --no-temporality + +run_with_base_args \ + --outlabel TXN-ALL-directed-topo-kinsub-temp \ + --network $TXN_ALL_UNDIRECTED \ + --partialmodel $KINSUB \ + --partialmodel $TXN_ALL_DIRECTED + +run_with_base_args \ + --outlabel TXN-1F-directed-topo-kinsub \ + --network $TXN_1F_UNDIRECTED \ + --partialmodel $KINSUB \ + --partialmodel $TXN_ALL_DIRECTED \ + --no-temporality + +run_with_base_args \ + --outlabel TXN-1F-directed-topo-kinsub-temp \ + --network $TXN_1F_UNDIRECTED \ + --partialmodel $KINSUB \ + --partialmodel $TXN_ALL_DIRECTED + +run_with_base_args \ + --outlabel TXN-ALL-undirected-topo-kinsub \ + --network $TXN_ALL_UNDIRECTED \ + --partialmodel $KINSUB \ + --no-temporality + +run_with_base_args \ + --outlabel TXN-ALL-undirected-topo-kinsub-temp \ + --network $TXN_ALL_UNDIRECTED \ + --partialmodel $KINSUB + +run_with_base_args \ + --outlabel TXN-1F-undirected-topo-kinsub \ + --network $TXN_1F_UNDIRECTED \ + --partialmodel $KINSUB \ + --no-temporality + +run_with_base_args \ + --outlabel TXN-1F-undirected-topo-kinsub-temp \ + --network $TXN_1F_UNDIRECTED \ + --partialmodel $KINSUB diff --git a/scripts/run-txn-graph-creation b/scripts/run-txn-graph-creation new file mode 100755 index 00000000..7a06af3d --- /dev/null +++ b/scripts/run-txn-graph-creation @@ -0,0 +1,6 @@ +#!/bin/bash + +TXNFILE="data/networks/comparison/TXN-ALL.edges" +sbt \ + "project tps-core" \ + "run-main tps.evaluation.txn.GraphCreation $TXNFILE 1.0" diff --git a/tps-core/lib/scalaz3.jar b/tps-core/lib/scalaz3.jar index 85c25fe4..fea9cc01 100644 Binary files a/tps-core/lib/scalaz3.jar and b/tps-core/lib/scalaz3.jar differ diff --git a/tps-core/src/main/scala/tps/Graphs.scala b/tps-core/src/main/scala/tps/Graphs.scala index 0395ccc7..20c269a2 100644 --- a/tps-core/src/main/scala/tps/Graphs.scala +++ b/tps-core/src/main/scala/tps/Graphs.scala @@ -1,7 +1,5 @@ package tps -import tps.util.LogUtils._ - object Graphs { case class Vertex(id: String) extends Serializable { diff --git a/tps-core/src/main/scala/tps/PeptideProteinMappingParser.scala b/tps-core/src/main/scala/tps/PeptideProteinMappingParser.scala index dfcc0a50..364460ef 100644 --- a/tps-core/src/main/scala/tps/PeptideProteinMappingParser.scala +++ b/tps-core/src/main/scala/tps/PeptideProteinMappingParser.scala @@ -1,7 +1,5 @@ package tps -import tps.util.FileUtils._ - object PeptideProteinMappingParser { def run(f: java.io.File): Map[String, Set[String]] = { val data = new TSVSource(f).data diff --git a/tps-core/src/main/scala/tps/ReferenceParser.scala b/tps-core/src/main/scala/tps/ReferenceParser.scala index bf1c6af1..2dfda283 100644 --- a/tps-core/src/main/scala/tps/ReferenceParser.scala +++ b/tps-core/src/main/scala/tps/ReferenceParser.scala @@ -7,10 +7,11 @@ object ReferenceParser { def run(f: java.io.File): (SignedDirectedGraph, Map[Edge, String]) = { val data = new TSVSource(f, noHeaders = false).data - var evidencePerEdge: Map[Edge, String] = Map.empty - val tuples = data.tuples.collect{ case tuple if tuple.size >= 7 => + val tuples = data.tuples.map{ tuple => + assert(tuple.size >= 6) + val Seq(src, tgt, lra, lri, rla, rli, rest @ _*) = tuple val edge = lexicographicEdge(src, tgt) @@ -21,6 +22,7 @@ object ReferenceParser { def labelValue(l: String): Boolean = l match { case "true" => true case "false" => false + case _ => sys.error(s"error parsing $l in $tuple") } if (labelValue(lra)) originalLabels += ActiveEdge(Forward, Activating) diff --git a/tps-core/src/main/scala/tps/SignedDirectedGraphOps.scala b/tps-core/src/main/scala/tps/SignedDirectedGraphOps.scala index 09972fdb..2b640bef 100644 --- a/tps-core/src/main/scala/tps/SignedDirectedGraphOps.scala +++ b/tps-core/src/main/scala/tps/SignedDirectedGraphOps.scala @@ -124,10 +124,64 @@ object SignedDirectedGraphOps { relevant.size } + def activeDirections( + labels: Set[SignedDirectedEdgeLabel] + ): Set[EdgeDirection] = { + labels collect { + case ActiveEdge(dir, sign) => dir + } + } + + def activeEdges(labels: Set[SignedDirectedEdgeLabel]): Set[ActiveEdge] = { + labels collect { + case ae: ActiveEdge => ae + } + } + + def sameUniqueDirection( + ls1: Set[SignedDirectedEdgeLabel], ls2: Set[SignedDirectedEdgeLabel] + ): Boolean = { + val activeDir1 = activeDirections(ls1) + val activeDir2 = activeDirections(ls2) + + activeDir1.size == 1 && activeDir2.size == 1 && activeDir1 == activeDir2 + } + + def sameUniqueActiveEdge( + ls1: Set[SignedDirectedEdgeLabel], ls2: Set[SignedDirectedEdgeLabel] + ): Boolean = { + val activeE1 = activeEdges(ls1) + val activeE2 = activeEdges(ls2) + + activeE1.size == 1 && activeE2.size == 1 && activeE1 == activeE2 + } + + def confictingUniqueActiveEdge( + ls1: Set[SignedDirectedEdgeLabel], ls2: Set[SignedDirectedEdgeLabel] + ): Boolean = { + val activeE1 = activeEdges(ls1) + val activeE2 = activeEdges(ls2) + + activeE1.size == 1 && activeE2.size == 1 && activeE1 != activeE2 + } + + def conflictingUniqueActiveDirection( + ls1: Set[SignedDirectedEdgeLabel], ls2: Set[SignedDirectedEdgeLabel] + ): Boolean = { + val activeDir1 = activeDirections(ls1) + val activeDir2 = activeDirections(ls2) + + activeDir1.size == 1 && activeDir2.size == 1 && activeDir1 != activeDir2 + } + def oneActiveDirection(es: Set[SignedDirectedEdgeLabel]): Boolean = { onlyForward(es) || onlyBackward(es) } + def oneActiveEdge(es: Set[SignedDirectedEdgeLabel]): Boolean = { + activeEdges(es).size == 1 + } + def onlyBackward(es: Set[SignedDirectedEdgeLabel]): Boolean = { canBeBackward(es) && !canBeForward(es) } diff --git a/tps-core/src/main/scala/tps/TimeSeries.scala b/tps-core/src/main/scala/tps/TimeSeries.scala index bd9e33ca..a17703c1 100644 --- a/tps-core/src/main/scala/tps/TimeSeries.scala +++ b/tps-core/src/main/scala/tps/TimeSeries.scala @@ -1,9 +1,5 @@ package tps -import Graphs._ - -import tps.util.StringUtils - case class Profile(id: String, values: Seq[Option[Double]]) case class TimeSeries( diff --git a/tps-core/src/main/scala/tps/evaluation/DirectedNetworkComparison.scala b/tps-core/src/main/scala/tps/evaluation/DirectedNetworkComparison.scala new file mode 100644 index 00000000..0efbfcf8 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/DirectedNetworkComparison.scala @@ -0,0 +1,255 @@ +package tps.evaluation + +import java.io.File + +import tps.Graphs.SignedDirectedEdgeLabel +import tps.Graphs.SignedDirectedGraph +import tps.TabularData +import tps.util.FileUtils +import tps.util.MathUtils +import tps.{SignedDirectedGraphOps, SignedDirectedGraphParser} + +import scala.collection.mutable.ListBuffer + +object DirectedNetworkComparison { + def main(args: Array[String]): Unit = { + // first argument: prior knowledge network + val priorKnowledgeFile = new File(args.head) + val priorKnowledgeNetwork = SignedDirectedGraphParser.run( + priorKnowledgeFile) + + // second argument: reference signed directed network + // rest of arguments: collection of signed directed networks to compare to + val allNetworkFiles = args.tail map (arg => new File(arg)) + val allNetworks = allNetworkFiles map ( + f => SignedDirectedGraphParser.run(f)) + + val firstNetwork = allNetworks.head + val restNetworks = allNetworks.tail + println(s"Comparing first network to ${restNetworks.size} networks.") + + val results = restNetworks map { n => + runComparison(firstNetwork, n, priorKnowledgeNetwork) + } + + printAggregateResults(results) + } + + def printAggregateResults( + results: Iterable[SignedDirectedGraphComparisonResult] + ): Unit = { + val percentiles = List(25, 50, 75) + + val header = List( + "Percentile", + "Edge type", + "Directed edges", + "Signed, directed edges" + ) + var rows = new ListBuffer[List[String]]() + + for (percentile <- percentiles) { + val percentileResults = aggregateResults(results, percentile) + + if (percentile == percentiles.head) { + rows.append(List( + "", + "Randomized networks with more edges", + percentileResults.directedComparison + .nbRunsWithMoreEdgesThanOriginal.toString, + percentileResults.signedDirectedComparison + .nbRunsWithMoreEdgesThanOriginal.toString + )) + } + + val directedResultsMap = percentileResults.directedComparison + .medianResults.asMap() + val signedDirectedResultsMap = percentileResults + .signedDirectedComparison.medianResults.asMap() + + for (field <- fields) { + val row = List( + percentile.toString, + field, + directedResultsMap(field).toString, + signedDirectedResultsMap(field).toString + ) + rows.append(row) + } + } + + FileUtils.writeToFile( + new File("analysis.tsv"), + TabularData(header, rows).toTSVString() + ) + } + + case class AggregateSignedDirectedGraphComparisonResult( + directedComparison: AggregateComparisonSubResult, + signedDirectedComparison: AggregateComparisonSubResult + ) + + case class SignedDirectedGraphComparisonResult( + directedComparison: ComparisonSubResult, + signedDirectedComparison: ComparisonSubResult + ) + + case class AggregateComparisonSubResult( + nbRunsWithMoreEdgesThanOriginal: Int, + medianResults: ComparisonSubResult + ) { + override def toString: String = { + s"Nb. runs with more edges: $nbRunsWithMoreEdgesThanOriginal\n" + + medianResults.toString() + } + } + + val unambigInOriginalField = "Unambiguous in the original network" + val unambigInAggregateField = "Median unambiguous in randomized networks" + val medianAgreeingUnamb = "Median agreeing unambiguous" + val medianConflictUnamb = "Median conflicting unambiguous" + val medianCommonPrior = "Median common prior knowledge" + val medianOnlyInOrig = "Median only in the original network" + val medianOnlyInAggr = "Median only in randomized networks" + + val fields = List( + unambigInOriginalField, + unambigInAggregateField, + medianAgreeingUnamb, + medianConflictUnamb, + medianCommonPrior, + medianOnlyInOrig, + medianOnlyInAggr + ) + + case class ComparisonSubResult( + nb1: Double, + nb2: Double, + nbCommon: Double, + nbCommonInPrior: Double, + nbAgreeing: Double, + nbConflicting: Double, + nbOnly1: Double, + nbOnly2: Double + ) { + def asMap(): Map[String, Double] = { + Map( + unambigInOriginalField -> nb1, + unambigInAggregateField -> nb2, + medianAgreeingUnamb -> nbAgreeing, + medianConflictUnamb -> nbConflicting, + medianCommonPrior -> nbCommonInPrior, + medianOnlyInOrig -> nbOnly1, + medianOnlyInAggr -> nbOnly2 + ) + } + } + + private def runComparison( + n1: SignedDirectedGraph, + n2: SignedDirectedGraph, + priorKnowledge: SignedDirectedGraph + ): SignedDirectedGraphComparisonResult = { + val directedComp = compareSubResults(n1, n2, priorKnowledge, + SignedDirectedGraphOps.oneActiveDirection, + SignedDirectedGraphOps.sameUniqueDirection, + SignedDirectedGraphOps.conflictingUniqueActiveDirection + ) + + val signedDirectedComp = compareSubResults(n1, n2, priorKnowledge, + SignedDirectedGraphOps.oneActiveEdge, + SignedDirectedGraphOps.sameUniqueActiveEdge, + SignedDirectedGraphOps.confictingUniqueActiveEdge + ) + + SignedDirectedGraphComparisonResult(directedComparison = directedComp, + signedDirectedComparison = signedDirectedComp) + } + + private def compareSubResults( + n1: SignedDirectedGraph, + n2: SignedDirectedGraph, + priorKnowledge: SignedDirectedGraph, + filter: Set[SignedDirectedEdgeLabel] => Boolean, + agreementFun: + (Set[SignedDirectedEdgeLabel], Set[SignedDirectedEdgeLabel]) => Boolean, + conflictFun: + (Set[SignedDirectedEdgeLabel], Set[SignedDirectedEdgeLabel]) => Boolean + ): ComparisonSubResult = { + val edges1 = n1.collect{ + case (e, ess) if filter(ess) => e + }.toSet + val edges2 = n2.collect{ + case (e, ess) if filter(ess) => e + }.toSet + + val commonE = edges1.intersect(edges2) + val commonEInPrior = commonE.intersect(priorKnowledge.keySet) + + val agreeing = commonE filter { e => + agreementFun(n1(e), n2(e)) + } + val conflicting = commonE filter { e => + conflictFun(n1(e), n2(e)) + } + + val only1 = edges1 -- edges2 + val only2 = edges2 -- edges1 + + ComparisonSubResult( + nb1 = edges1.size, + nb2 = edges2.size, + nbCommon = commonE.size, + nbCommonInPrior = commonEInPrior.size, + nbAgreeing = agreeing.size, + nbConflicting = conflicting.size, + nbOnly1 = only1.size, + nbOnly2 = only2.size + ) + } + + private def aggregateResults( + results: Iterable[SignedDirectedGraphComparisonResult], + percentile: Int + ): AggregateSignedDirectedGraphComparisonResult = { + AggregateSignedDirectedGraphComparisonResult( + directedComparison = + aggregateSubResults(results.map(_.directedComparison), percentile), + signedDirectedComparison = + aggregateSubResults(results.map(_.signedDirectedComparison), percentile) + ) + } + + private def aggregateSubResults( + results: Iterable[ComparisonSubResult], + percentile: Int + ): AggregateComparisonSubResult = { + def aggregation(xs: Iterable[Double]) = { + MathUtils.percentile(xs, percentile) + } + + val aggregatedResult = ComparisonSubResult( + aggregation(results.map(_.nb1)), + aggregation(results.map(_.nb2)), + aggregation(results.map(_.nbCommon)), + aggregation(results.map(_.nbCommonInPrior)), + aggregation(results.map(_.nbAgreeing)), + aggregation(results.map(_.nbConflicting)), + aggregation(results.map(_.nbOnly1)), + aggregation(results.map(_.nbOnly2)) + ) + + AggregateComparisonSubResult( + countRunsWithMoreEdges(results), + aggregatedResult + ) + } + + private def countRunsWithMoreEdges( + results: Iterable[ComparisonSubResult] + ): Int = { + val nb1 = results.head.nb1 + val nb2s = results.map(_.nb2) + nb2s.count(nb2 => nb2 > nb1) + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/NetworkReferenceComparison.scala b/tps-core/src/main/scala/tps/evaluation/NetworkReferenceComparison.scala new file mode 100644 index 00000000..71905db9 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/NetworkReferenceComparison.scala @@ -0,0 +1,190 @@ +package tps.evaluation + +import tps._ +import tps.Graphs._ +import tps.SignedDirectedGraphOps._ +import parsing.DBNNetworkParser +import java.io.File + +import tps.util.MathUtils + +object NetworkReferenceComparison { + def main(args: Array[String]): Unit = { + // first argument: reference network + val referenceFile = new File(args(0)) + val referenceName = referenceFile.getName() + val (referenceNetwork, referenceEvidences) = ReferenceParser.run( + referenceFile) + + // rest of arguments: network files to use for average/median analysis + val networkFiles = args.tail map (a => new File(a)) + val networks = networkFiles map (f => SignedDirectedGraphParser.run(f)) + println(s"Parsed ${networks.size} networks.") + + val results = networks.zipWithIndex map { + case (network, i) => { + compare(network, referenceNetwork, s"network-$i", referenceName) + } + } + println(results.mkString("\n")) + + // compute aggregate metrics + val aggregateResult = aggregateResults(results) + println(aggregateResult) + } + + def resultString(res: NetworkReferenceComparisonResult): String = { + ??? + } + + def aggregateResults( + results: Iterable[NetworkReferenceComparisonResult] + ): NetworkReferenceComparisonResult = { + assert(results.forall(_.referenceName == results.head.referenceName)) + + NetworkReferenceComparisonResult( + "aggregate-results", + results.head.referenceName, + MathUtils.median(results.map(_.nbCandidateEdges)), + results.head.nbReferenceEdges, + MathUtils.median(results.map(_.nbCommonEdges)), + MathUtils.average(results.map(_.undirectedPrecision)), + MathUtils.average(results.map(_.undirectedRecall)), + MathUtils.median(results.map(_.nbDirectedEdges)), + MathUtils.median(results.map(_.directedEdgeRatio)), + MathUtils.median(results.map(_.nbCommonDirectedEdges)), + MathUtils.median(results.map(_.commonDirectedEdgeRatio)), + MathUtils.median(results.map(_.nbMatchingDirectionEdges)), + MathUtils.median(results.map(_.matchingDirectionEdgeRatio)), + MathUtils.median(results.map(_.nbConflictingDirectionEdges)), + MathUtils.median(results.map(_.conflictingDirectionEdgeRatio)), + MathUtils.median(results.map(_.nbUnconfirmedDirectionEdges)), + MathUtils.median(results.map(_.unconfirmedDirectionEdgeRatio)) + ) + } + + // legacy analysis + def runComparativeAnalysis(refFn: String): Unit = { + val refFile = new File(refFn) + val refName = refFile.getName() + val (refNetwork, refEvidence) = ReferenceParser.run(refFile) + + val networkFolder = new File("data/networks/evaluation") + + val dbnFn = s"$networkFolder/DBN.tsv" + val dbnFile = new File(dbnFn) + val dbnMinProb = 0.025 + val dbnNetwork = DBNNetworkParser.run(dbnFile, dbnMinProb) + + // PIN + kin-sub edges + val pinFn = "data/networks/directed-pin-with-resource-edges.tsv" + val pinFile = new File(pinFn) + val pin = toSignedDirectedGraph(PINParser.run(pinFile)) + + // SIF networks + // undirected TXN + TPS (time series + kinase-substrate) + val sifFiles = networkFolder.listFiles() filter { f => + f.getName().endsWith(".sif") + } + val sifNetworks = sifFiles map { f => + f.getName() -> SignedDirectedGraphParser.run(f) + } + + val candidates = Map( + "PIN" -> pin, + "DBN" -> dbnNetwork + ) ++ sifNetworks + + for ((id, network) <- candidates) { + println(resultString(compare(network, refNetwork, id, refName))) + } + } + + // Use doubles instead of integers to compute aggregate statistics + case class NetworkReferenceComparisonResult( + candidateName: String, + referenceName: String, + nbCandidateEdges: Double, + nbReferenceEdges: Double, + nbCommonEdges: Double, + undirectedPrecision: Double, + undirectedRecall: Double, + nbDirectedEdges: Double, + directedEdgeRatio: Double, + nbCommonDirectedEdges: Double, + commonDirectedEdgeRatio: Double, + nbMatchingDirectionEdges: Double, + matchingDirectionEdgeRatio: Double, + nbConflictingDirectionEdges: Double, + conflictingDirectionEdgeRatio: Double, + nbUnconfirmedDirectionEdges: Double, + unconfirmedDirectionEdgeRatio: Double + ) + + def compare( + candidate: SignedDirectedGraph, + reference: SignedDirectedGraph, + candidateName: String, + referenceName: String + ) = { + // compute precision and recall where relevance is whether a selected edge + // is in the reference + + val candidateE = candidate.keySet + val referenceE = reference.keySet + + val commonE = candidateE intersect referenceE + var undirPrecision = commonE.size.toDouble / candidateE.size.toDouble + if (candidateE.isEmpty) undirPrecision = 0.0 + val undirRecall = commonE.size.toDouble / referenceE.size.toDouble + + val directedE = candidateE filter { e => + oneActiveDirection(candidate(e)) + } + + val commonDirectedE = commonE filter { e => + oneActiveDirection(candidate(e)) + } + + val matchingDirectionE = commonE filter { e => + val candidateL = candidate(e) + val referenceL = reference(e) + + onlyForward(candidateL) && onlyForward(referenceL) || + onlyBackward(candidateL) && onlyBackward(referenceL) + } + + val conflictingE = commonE filter { e => + val candidateL = candidate(e) + val referenceL = reference(e) + + onlyForward(candidateL) && onlyBackward(referenceL) || + onlyBackward(candidateL) && onlyForward(referenceL) + } + + val unconfirmedDirectionE = commonE filter { e => + oneActiveDirection(candidate(e)) && ambiguousDirection(reference(e)) + } + + NetworkReferenceComparisonResult( + candidateName, + referenceName, + candidateE.size, + referenceE.size, + commonE.size, + undirPrecision, + undirRecall, + directedE.size, + directedE.size.toDouble / candidateE.size, + commonDirectedE.size, + commonDirectedE.size.toDouble / directedE.size.toDouble, + matchingDirectionE.size, + matchingDirectionE.size.toDouble / directedE.size.toDouble, + conflictingE.size, + conflictingE.size.toDouble / directedE.size.toDouble, + unconfirmedDirectionE.size, + unconfirmedDirectionE.size.toDouble / directedE.size.toDouble + ) + } + +} diff --git a/tps-core/src/main/scala/tps/evaluation/OverlapAnalysis.scala b/tps-core/src/main/scala/tps/evaluation/OverlapAnalysis.scala new file mode 100644 index 00000000..0254e1d9 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/OverlapAnalysis.scala @@ -0,0 +1,88 @@ +package tps.evaluation + +import tps._ +import tps.Graphs._ +import tps.synthesis._ + +import java.io.File + +object OverlapAnalysis { + def main(args: Array[String]): Unit = { + // get reference + val refFile = new File(args(0)) + val refName = refFile.getName() + val (refNetwork, refEvidence) = ReferenceParser.run(refFile) + + // get time series + val tsFile = new File(args(1)) + val timeSeries = TimeSeriesParser.run(tsFile) + + // get mapping + val mapFile = new File(args(2)) + val mapping = PeptideProteinMappingParser.run(mapFile) + + // first and prev scores + val firstScores = TimeSeriesScoresParser.run(new File(args(3))) + val prevScores = TimeSeriesScoresParser.run(new File(args(4))) + + val significanceThreshold = 0.01 + + // get time series interpretation + val interp = new TriggerInterpretation( + useMonotonicity = true, + graph = UndirectedGraphOps.emptyGraph, + ts = timeSeries, + firstScores = firstScores, + prevScores = prevScores, + threshold = significanceThreshold + ) + + val significantTimeSeries = timeSeries.copy( + profiles = timeSeries.profiles filter (p => + Synthesis.profileIsSignificant(p, firstScores, prevScores, significanceThreshold) + ) + ) + println(s"There are ${significantTimeSeries.profiles.size} significant " + + s"profiles") + + val refProtNames = refNetwork.keySet.flatMap { + case Edge(Vertex(id1), Vertex(id2)) => Set(id1, id2) + } + println(s"There are ${refProtNames.size} names to check in the network.") + + for ((label, i) <- significantTimeSeries.labels.zipWithIndex) { + val protsChangingAtStep = significantTimeSeries.profiles.flatMap{ p => + proteinsWithSignificantChange(p, i, interp, mapping) + }.toSet + + val refProtsChangingAtStep = protsChangingAtStep.intersect(refProtNames) + val ratioOfRefOverlap = refProtsChangingAtStep.size.toDouble / + protsChangingAtStep.size + val row = List( + refName, + label, + protsChangingAtStep.size, + refProtsChangingAtStep.size, + ratioOfRefOverlap + ) + println(row.mkString("\t")) + } + + } + + def proteinsWithSignificantChange( + profile: Profile, + step: Int, + interpretation: TriggerInterpretation, + peptideToProteins: Map[String, Set[String]] + ): Set[String] = { + val profileProteins = peptideToProteins(profile.id) + if (interpretation.allowedActivationIntervals(profile).contains(step) || + interpretation.allowedInhibitionIntervals(profile).contains(step)) { + profileProteins + } else { + Set[String]() + } + } + +} diff --git a/tps-core/src/main/scala/tps/evaluation/SignificanceScoreAndPrizeProducer.scala b/tps-core/src/main/scala/tps/evaluation/SignificanceScoreAndPrizeProducer.scala new file mode 100644 index 00000000..b8d981c5 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/SignificanceScoreAndPrizeProducer.scala @@ -0,0 +1,193 @@ +package tps.evaluation + +import java.io.File + +import tps.PeptideExpansion.PeptideProteinMap +import tps.PeptideProteinMappingParser +import tps.Profile +import tps.TabularData +import tps.TimeSeries +import tps.TimeSeriesParser +import tps.util.FileUtils + +/** + * Generates significance score and protein prize files based on fold change + * thresholding from a time series file with a single replicate. + */ +object SignificanceScoreAndPrizeProducer { + + val SIGNIFICANT_SCORE = 0.0 + val NON_SIGNIFICANT_SCORE = 1.0 + + def main(args: Array[String]): Unit = { + val timeSeriesFile = new File(args(0)) + val mappingFile = new File(args(1)) + val threshold = args(2).toDouble + val timeSeries = TimeSeriesParser.run(timeSeriesFile) + val peptideProteinMap = PeptideProteinMappingParser.run(mappingFile) + + // compute fold changes + var firstFCs = firstFoldChanges(timeSeries) + var prevFCs = prevFoldChanges(timeSeries) + + // find max fold change to replace NaN and infinity values with it + val maxFC = math.max( + maxNonInfNonNaNValue(firstFCs.values.flatten), + maxNonInfNonNaNValue(prevFCs.values.flatten) + ) + firstFCs = replaceInfAndNan(firstFCs, maxFC) + prevFCs = replaceInfAndNan(prevFCs, maxFC) + + // convert fold changes to scores + val firstScores = scoresFromFoldChanges(firstFCs, threshold) + val prevScores = scoresFromFoldChanges(prevFCs, threshold) + + // save scores to file + saveScores(firstScores, firstScoreLabels(timeSeries), "first-scores.tsv") + saveScores(prevScores, prevScoreLabels(timeSeries), "prev-scores.tsv") + + // take max fold change per peptide as prize + val pepPrizes = peptidePrizes(firstFCs, prevFCs) + + // take max peptide prize for each protein + val protPrizes = collapsePrizes(pepPrizes, peptideProteinMap) + savePrizes(protPrizes) + } + + private def maxNonInfNonNaNValue(xs: Iterable[Double]): Double = { + xs.filter(x => !x.isNaN && !x.isInfinity).max + } + + private def replaceInfAndNan( + foldChanges: Map[String, Seq[Double]], + maxFoldChange: Double + ): Map[String, Seq[Double]] = { + def replace(x: Double): Double = { + if (x.isNaN || x.isInfinity) maxFoldChange else x + } + + foldChanges map { + case (id, vs) => id -> vs.map(replace) + } + } + + private def saveScores( + scores: Map[String, Seq[Double]], labels: Seq[String], fname: String + ): Unit = { + val tuples = scores map { + case (id, vs) => id +: vs.map(_.toString) + } + val tsvString = TabularData(labels, tuples.toSeq).toTSVString() + FileUtils.writeToFile(new File(fname), tsvString) + } + + private def savePrizes(proteinPrizes: Map[String, Double]): Unit = { + val tuples = proteinPrizes map { + case (id, prize) => List(id, prize.toString) + } + val tsvString = TabularData(Nil, tuples.toSeq).toTSVString( + printHeaders = false) + FileUtils.writeToFile(new File("protein-prizes.tsv"), tsvString) + } + + private def collapsePrizes( + peptidePrizes: Map[String, Double], peptideProteinMap: PeptideProteinMap + ): Map[String, Double] = { + val proteinPeptideMap = reverseMap(peptideProteinMap) + for ((prot, peps) <- proteinPeptideMap) yield { + val maxPepPrize = peps.map(pep => peptidePrizes(pep)).max + prot -> maxPepPrize + } + } + + private def firstFoldChanges(ts: TimeSeries): Map[String, Seq[Double]] = { + ts.profiles.map{ + case Profile(id, vs) => { + val baseline = vs.head.get + val restValues = vs.tail + val foldChanges = restValues map ( vOpt => + absLog2FoldChange(vOpt.get, baseline)) + id -> foldChanges + } + }.toMap + } + + private def prevFoldChanges(ts: TimeSeries): Map[String, Seq[Double]] = { + ts.profiles.map{ + case Profile(id, vs) => { + val foldChanges = vs.zip(vs.tail).map { + case (prevVal, currVal) => absLog2FoldChange(currVal.get, prevVal.get) + } + id -> foldChanges + } + }.toMap + } + + private def peptidePrizes( + firstFoldChanges: Map[String, Seq[Double]], + prevFoldChanges: Map[String, Seq[Double]] + ): Map[String, Double] = { + assert(firstFoldChanges.keySet == prevFoldChanges.keySet) + firstFoldChanges.keySet.map{ id => + id -> math.max(firstFoldChanges(id).max, prevFoldChanges(id).max) + }.toMap + } + + private def reverseMap( + ppm: PeptideProteinMap + ): Map[String, Set[String]] = { + var proteinPeptideMap = Map[String, Set[String]]() + for ((pep, prots) <- ppm) { + for (prot <- prots) { + proteinPeptideMap.get(prot) match { + case Some(peps) => { + proteinPeptideMap += prot -> (peps + pep) + } + case None => { + proteinPeptideMap += prot -> Set(pep) + } + } + } + } + proteinPeptideMap + } + + private def firstScoreLabels(ts: TimeSeries): Seq[String] = { + val baselineLabel = ts.labels.head + val restLabels = ts.labels.tail + val scoreLabels = restLabels map { l => + s"$l vs. $baselineLabel" + } + "id" +: scoreLabels + } + + private def prevScoreLabels(ts: TimeSeries): Seq[String] = { + val scoreLabels = ts.labels.zip(ts.labels.tail).map { + case (prevLabel, currLabel) => + s"$currLabel vs. $prevLabel" + } + "id" +: scoreLabels + } + + private def absLog2FoldChange(value: Double, baseline: Double): Double = { + math.abs(math.log(value / baseline) / math.log(2)) + } + + private def scoresFromFoldChanges( + peptideToFoldChanges: Map[String, Seq[Double]], + threshold: Double + ): Map[String, Seq[Double]] = { + peptideToFoldChanges map { + case (id, fcs) => { + val scores = fcs map { fc => + if (fc >= threshold) { + SIGNIFICANT_SCORE + } else { + NON_SIGNIFICANT_SCORE + } + } + id -> scores + } + } + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/TimeSeriesPermutation.scala b/tps-core/src/main/scala/tps/evaluation/TimeSeriesPermutation.scala new file mode 100644 index 00000000..9baa7078 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/TimeSeriesPermutation.scala @@ -0,0 +1,69 @@ +package tps.evaluation + +import java.io.File + +import tps.FileReporter +import tps.Profile +import tps.TimeSeriesParser +import tps.TimeSeriesScoresParser +import tps.printing.TimeSeriesPrinter +import tps.printing.TimeSeriesScoresPrinter +import tps.util.CollectionUtils + +import scala.util.Random + +object TimeSeriesPermutation { + def main(args: Array[String]): Unit = { + val timeSeries = TimeSeriesParser.run(new File(args(0))) + val firstScores = TimeSeriesScoresParser.run(new File(args(1))) + val prevScores = TimeSeriesScoresParser.run(new File(args(2))) + val seed = args(3).toInt + val outFolder = new File(args(4)) + + val rand = new Random(seed) + val permuteFun = permute(rand) _ + + + var permutedProfiles = Set[Profile]() + var permutedFirstScores = Map[String, Seq[Double]]() + var permutedPrevScores = Map[String, Seq[Double]]() + + for (profile <- timeSeries.profiles) { + val (permutedProfile, permutedFirst, permutedPrev) = permuteFun( + profile, firstScores(profile.id), prevScores(profile.id)) + permutedProfiles += permutedProfile + permutedFirstScores += profile.id -> permutedFirst + permutedPrevScores += profile.id -> permutedPrev + } + + val permutedTimeSeries = timeSeries.copy( + profiles = permutedProfiles.toSeq.sortBy(_.id)) + + val reporter = new FileReporter(outFolder, Some("permuted")) + reporter.output("time-series.tsv", + TimeSeriesPrinter(permutedTimeSeries)) + reporter.output("first-scores.tsv", + TimeSeriesScoresPrinter(permutedFirstScores)) + reporter.output("prev-scores.tsv", + TimeSeriesScoresPrinter(permutedPrevScores)) + } + + private def permute(rand: Random)( + profile: Profile, firstScores: Seq[Double], prevScores: Seq[Double] + ): (Profile, Seq[Double], Seq[Double]) = { + val n = profile.values.size - 1 + val permutation = rand.shuffle((0 until n).toList) + + // permute tail of profile values + val permutedTailValues = CollectionUtils.permuteSeq(profile.values.tail, + permutation) + val permutedProfile = profile.copy( + values = profile.values.head +: permutedTailValues) + + ( + permutedProfile, + CollectionUtils.permuteSeq(firstScores, permutation), + CollectionUtils.permuteSeq(prevScores, permutation) + ) + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/UndirectedNetworkComparison.scala b/tps-core/src/main/scala/tps/evaluation/UndirectedNetworkComparison.scala new file mode 100644 index 00000000..62b560b3 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/UndirectedNetworkComparison.scala @@ -0,0 +1,48 @@ +package tps.evaluation + +import java.io.File + +import tps.Graphs.UndirectedGraph +import tps.UndirectedGraphParser + +object UndirectedNetworkComparison { + def main(args: Array[String]): Unit = { + val n1File = new File(args(0)) + val n2File = new File(args(1)) + + val network1 = UndirectedGraphParser.run(n1File) + val network2 = UndirectedGraphParser.run(n2File) + + println(s"Network 1: ${n1File.getPath}") + println(s"Network 2: ${n2File.getPath}") + + undirectedNetworkComparison(network1, network2) + } + + def undirectedNetworkComparison( + network1: UndirectedGraph, + network2: UndirectedGraph + ): Unit = { + val commonV = network1.V intersect network2.V + val commonE = network1.E intersect network2.E + + val exclusiveV1 = network1.V -- network2.V + val exclusiveV2 = network2.V -- network1.V + + val exclusiveE1 = network1.E -- network2.E + val exclusiveE2 = network2.E -- network1.E + + println(s"Common V: ${commonV.size}") + println(s"Common E: ${commonE.size}") + + println(s"Only in V1: ${exclusiveV1.size}") + println(exclusiveV1.mkString("\n")) + println(s"Only in V2: ${exclusiveV2.size}") + println(exclusiveV2.mkString("\n")) + + println(s"Only in E1: ${exclusiveE1.size}") + println(exclusiveE1.mkString("\n")) + println(s"Only in E2: ${exclusiveE2.size}") + println(exclusiveE2.mkString("\n")) + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/dbn/MatrixGraphConversion.scala b/tps-core/src/main/scala/tps/evaluation/dbn/MatrixGraphConversion.scala new file mode 100644 index 00000000..10ab56f0 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/dbn/MatrixGraphConversion.scala @@ -0,0 +1,82 @@ +package tps.evaluation.dbn + +import tps._ +import tps.Graphs._ + +object MatrixGraphConversion { + def main(args: Array[String]): Unit = { + val opts = ArgHandling.parseOptions(args) + + val pinPath = new java.io.File("data/networks/directed-pin-with-resource-edges.tsv") + val pin = PINParser.run(pinPath) + val timeSeries = TimeSeriesParser.run(opts.timeSeriesPath) + val peptideProteinMap = opts.peptideProteinMapPath map { p => + PeptideProteinMappingParser.run(p) + } getOrElse Map.empty + val reporter = new FileReporter(opts.outFolder, opts.outLabel) + + readGraphFromMatrix(timeSeries, peptideProteinMap, reporter) + } + + def createAdjacencyMatrix( + pin: DirectedGraph, + timeSeries: TimeSeries, + peptideProteinMap: Map[String, Set[String]], + reporter: FileReporter + ): Unit = { + // TODO could add to CLI arguments + val adjMatrix = AdjacencyMatrix.create(timeSeries, pin, peptideProteinMap) + val tsvContent = adjMatrix.toTabularData.toTSVString(printHeaders = false) + + reporter.output("adjacencyMatrix.tsv", tsvContent) + } + + def readGraphFromMatrix( + timeSeries: TimeSeries, + peptideProteinMap: Map[String, Set[String]], + reporter: FileReporter + ) = { + val edgeProbabilityFile = new java.io.File("edge_prob_matrix.tsv") + val interactionSignFile = new java.io.File("interaction_sign_matrix.tsv") + + val edgeProbData = new TSVSource(edgeProbabilityFile, noHeaders = true).data + val interactionSignData = new TSVSource(interactionSignFile, noHeaders = true).data + + val edgeProbMatrix = edgeProbData.tuples.toIndexedSeq.map(_.toIndexedSeq) + val interactionSignMatrix = interactionSignData.tuples.toIndexedSeq.map(_.toIndexedSeq) + + var labels = timeSeries.profiles.map(_.id).toIndexedSeq + + // TODO handle self edges + val tuples = for { + (peptide1, i) <- labels.zipWithIndex + (peptide2, j) <- labels.zipWithIndex + if i != j + } yield { + val prots1 = peptideProteinMap.getOrElse(peptide1, Set[String]()) + val prots2 = peptideProteinMap.getOrElse(peptide2, Set[String]()) + + assert(prots1.size == 1, s"prots for $peptide1: $prots1") + assert(prots2.size == 1, s"prots for $peptide2: $prots2") + + val prob = edgeProbMatrix(i)(j) + val sign = interactionSignMatrix(i)(j) + + // TODO adjust direction in terms of protein lexicographic order + val edgeTuples = for { + prot1 <- prots1 + prot2 <- prots2 + if prot1 != prot2 + } yield { + Seq(prot1, prot2, prob, sign, peptide1, peptide2) + } + + edgeTuples + } + + val fields = Seq("protein 1", "protein 2", "probability", "sign", "peptide 1", "peptide 2") + val outputData = TabularData(fields, tuples.flatten) + + reporter.output("matrix_network.tsv", outputData.toTSVString()) + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisq2SIFConverter.scala b/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisq2SIFConverter.scala new file mode 100644 index 00000000..f59ccba7 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisq2SIFConverter.scala @@ -0,0 +1,40 @@ +package tps.evaluation.funchisq + +import java.io.File + +import tps.PeptideProteinMappingParser +import tps.evaluation.parsing.FunChisqParser + +object FunChisq2SIFConverter { + + def main(args: Array[String]): Unit = { + // parse arguments + val networkFile = new File(args(0)) + val mappingFile = new File(args(1)) + val pValueThreshold = args(2).toDouble + val maxNbEdges = args(3).toInt + + // read FC output + var funChisqGraph = FunChisqParser.run(networkFile) + + // read mapping + val mapping = PeptideProteinMappingParser.run(mappingFile) + + // map nodes + funChisqGraph = FunChisqGraphs.mapNodes(funChisqGraph, mapping) + + // filter by given p-value threshold + funChisqGraph = funChisqGraph filter { + case (_, score) => score.pValue <= pValueThreshold + } + + // order by decreasing statistic and limit to first N edges + val strongestEdges = funChisqGraph.sortBy{ + case (_, score) => score.statistic + }.reverse.take(maxNbEdges) + + // print in SIF format + println(FunChisqSIFPrinter.print(strongestEdges)) + } + +} diff --git a/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisqGraphs.scala b/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisqGraphs.scala new file mode 100644 index 00000000..a4972f5a --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisqGraphs.scala @@ -0,0 +1,39 @@ +package tps.evaluation.funchisq + +import tps.Graphs.Edge +import tps.Graphs.Vertex + +object FunChisqGraphs { + + // No lexicographic order imposed on edges + type FunChisqGraph = Seq[(Edge, FunChisqScore)] + case class FunChisqScore(statistic: Double, pValue: Double) + + def mapNodes( + g: FunChisqGraph, mapping: Map[String, Set[String]] + ): FunChisqGraph = { + // flatmap to all protein-level edges + val mappedPairSeq = g flatMap { + case (edge, score) => { + for { + mappedV1 <- mapping(edge.v1.id) + mappedV2 <- mapping(edge.v2.id) + } yield { + val mappedEdge = Edge( + Vertex(mappedV1), + Vertex(mappedV2) + ) + (mappedEdge, score) + } + } + } + + // consolidate duplicate edges (take edge with minimum p-value) + val groupedPairs = mappedPairSeq.groupBy(_._1) + groupedPairs.toSeq map { + case (edge, pairSet) => { + pairSet.minBy(_._2.pValue) + } + } + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisqSIFPrinter.scala b/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisqSIFPrinter.scala new file mode 100644 index 00000000..415b1579 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/funchisq/FunChisqSIFPrinter.scala @@ -0,0 +1,18 @@ +package tps.evaluation.funchisq + +import tps.evaluation.funchisq.FunChisqGraphs.FunChisqGraph + +object FunChisqSIFPrinter { + def print(fcg: FunChisqGraph): String = { + val sb = new StringBuilder() + + for ((edge, _) <- fcg) { + val rel = "N" + + sb append List(edge.v1.id, rel, edge.v2.id).mkString("\t") + sb append "\n" + } + + sb.toString() + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/parsing/DBNNetworkParser.scala b/tps-core/src/main/scala/tps/evaluation/parsing/DBNNetworkParser.scala new file mode 100644 index 00000000..856b6873 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/parsing/DBNNetworkParser.scala @@ -0,0 +1,31 @@ +package tps.evaluation.parsing + +import tps.Graphs._ +import tps.GraphParsing._ +import tps.TSVSource + +object DBNNetworkParser { + def run(f: java.io.File, minProb: Double): SignedDirectedGraph = { + val data = new TSVSource(f, noHeaders = false).data + + val tuplesWithLabels: Seq[(Edge, SignedDirectedEdgeLabel, Double)] = data.tuples.map{ + case Seq(p1, p2, prob, sign, pep1, pep2) => { + val edgeSign = sign match { + case "-1" => Inhibiting + case "1" => Activating + } + val edge = lexicographicEdge(p1, p2) + val edgeDir = lexicographicForwardDirection(p1, p2) + val label = ActiveEdge(edgeDir, edgeSign) + (edge, label, prob.toDouble) + } + } + + val tuplesAboveThreshold = tuplesWithLabels collect { + case (edge, label, prob) if prob >= minProb => edge -> Set[SignedDirectedEdgeLabel](label) + } + + aggregateLabels(tuplesAboveThreshold) + } +} + diff --git a/tps-core/src/main/scala/tps/evaluation/parsing/FunChisqParser.scala b/tps-core/src/main/scala/tps/evaluation/parsing/FunChisqParser.scala new file mode 100644 index 00000000..e626412f --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/parsing/FunChisqParser.scala @@ -0,0 +1,27 @@ +package tps.evaluation.parsing + +import java.io.File + +import tps.Graphs.Edge +import tps.Graphs.Vertex +import tps.TSVSource +import tps.evaluation.funchisq.FunChisqGraphs.FunChisqGraph +import tps.evaluation.funchisq.FunChisqGraphs.FunChisqScore + +object FunChisqParser { + def run(f: File): FunChisqGraph = { + val data = new TSVSource(f, noHeaders = false).data + assert(data.fields == Seq("source", "target", "statistic", "p-value")) + + val pairs = data.tuples map { + case Seq(p1, p2, statString, pValString) => { + val edge = Edge(Vertex(p1), Vertex(p2)) + val statistic = statString.toDouble + val pValue = pValString.toDouble + (edge, FunChisqScore(statistic, pValue)) + } + } + + pairs + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/parsing/TXNParser.scala b/tps-core/src/main/scala/tps/evaluation/parsing/TXNParser.scala new file mode 100644 index 00000000..4ec59705 --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/parsing/TXNParser.scala @@ -0,0 +1,32 @@ +package tps.evaluation.parsing + +import tps.Graphs._ +import tps.GraphParsing._ +import tps.TSVSource + +object TXNParser { + def run(fn: String, minFlow: Double): SignedDirectedGraph = { + val f = new java.io.File(fn) + val data = new TSVSource(f, noHeaders = true).data + + val tuples = data.tuples.map{ + case Seq(edgeStr, labelStr, flowStr) => { + val Seq(id1, id2) = edgeStr.split("~").toSeq + val flow = flowStr.toDouble + val edge = lexicographicEdge(id1, id2) + val edgeDir = lexicographicForwardDirection(id1, id2) + val labels = Set[SignedDirectedEdgeLabel]( + ActiveEdge(edgeDir, Activating), + ActiveEdge(edgeDir, Inhibiting) + ) + (edge, labels, flow) + } + } + + val tuplesAboveMinFlow = tuples collect { + case (edge, labels, flow) if flow >= minFlow => edge -> labels + } + + aggregateLabels(tuplesAboveMinFlow) + } +} diff --git a/tps-core/src/main/scala/tps/evaluation/txn/GraphCreation.scala b/tps-core/src/main/scala/tps/evaluation/txn/GraphCreation.scala new file mode 100644 index 00000000..09ea502f --- /dev/null +++ b/tps-core/src/main/scala/tps/evaluation/txn/GraphCreation.scala @@ -0,0 +1,21 @@ +package tps.evaluation.txn + +import tps._ +import tps.Graphs._ + +import tps.evaluation.parsing.TXNParser + +object GraphCreation { + def main(args: Array[String]): Unit = { + val txnFn = args(0) + val minFlow = args(1).toDouble + + val network = TXNParser.run(txnFn, minFlow) + + val outFile = new java.io.File(s"TXN-flow-$minFlow.sif") + tps.util.FileUtils.writeToFile( + outFile, + SIFPrinter.print(network) + ) + } +} diff --git a/tps-core/src/main/scala/tps/printing/TimeSeriesPrinter.scala b/tps-core/src/main/scala/tps/printing/TimeSeriesPrinter.scala new file mode 100644 index 00000000..eac70f70 --- /dev/null +++ b/tps-core/src/main/scala/tps/printing/TimeSeriesPrinter.scala @@ -0,0 +1,20 @@ +package tps.printing + +import tps.TabularData +import tps.TimeSeries + +object TimeSeriesPrinter { + private val ID_FIELD = "id" + + def apply(ts: TimeSeries): String = { + val fields = ID_FIELD +: ts.labels + val tuples = ts.profiles.map(p => p.id +: p.values.map(optDouble2String)) + + TabularData(fields, tuples).toTSVString() + } + + private def optDouble2String(dOpt: Option[Double]): String = dOpt match { + case Some(v) => v.toString() + case None => "NA" + } +} diff --git a/tps-core/src/main/scala/tps/printing/TimeSeriesScoresPrinter.scala b/tps-core/src/main/scala/tps/printing/TimeSeriesScoresPrinter.scala new file mode 100644 index 00000000..ca3a0368 --- /dev/null +++ b/tps-core/src/main/scala/tps/printing/TimeSeriesScoresPrinter.scala @@ -0,0 +1,14 @@ +package tps.printing + +import tps.TabularData + +object TimeSeriesScoresPrinter { + + def apply(scores: Map[String, Seq[Double]]): String = { + val tuples = scores.toSeq map { + case (id, vs) => id +: vs.map(_.toString()) + } + TabularData(Nil, tuples).toTSVString(printHeaders = false) + } + +} diff --git a/tps-core/src/main/scala/tps/util/CollectionUtils.scala b/tps-core/src/main/scala/tps/util/CollectionUtils.scala index 479f8529..d1cce275 100644 --- a/tps-core/src/main/scala/tps/util/CollectionUtils.scala +++ b/tps-core/src/main/scala/tps/util/CollectionUtils.scala @@ -6,4 +6,8 @@ object CollectionUtils { case List(x) => Nil case x :: xs => xs.map((x, _)) ::: pairAllToSuccessors(xs) } + + def permuteSeq[T](xs: Seq[T], permutation: Seq[Int]): Seq[T] = { + permutation map (i => xs(i)) + } } diff --git a/tps-core/src/main/scala/tps/util/FileUtils.scala b/tps-core/src/main/scala/tps/util/FileUtils.scala index 10ec9c2d..f2d75243 100644 --- a/tps-core/src/main/scala/tps/util/FileUtils.scala +++ b/tps-core/src/main/scala/tps/util/FileUtils.scala @@ -19,10 +19,8 @@ object FileUtils { } } - def lines(f: File): List[String] = lines(f.getAbsolutePath) - def lines(fname: String): List[String] = { - - val br = new BufferedReader(new FileReader(fname)) + def lines(f: File): List[String] = { + val br = new BufferedReader(new FileReader(f.getAbsolutePath())) var lines: List[String] = Nil try { var line = br.readLine(); @@ -38,9 +36,8 @@ object FileUtils { } } - def uncommentedLines(f: File): List[String] = uncommentedLines(f.getAbsolutePath) - def uncommentedLines(fname: String): List[String] = { - lines(fname) filter (!_.startsWith("#")) + def uncommentedLines(f: File): List[String] = { + lines(f) filter (!_.startsWith("#")) } def fileExists(fn: String): Boolean = new File(fn).isFile diff --git a/tps-core/src/main/scala/tps/util/MathUtils.scala b/tps-core/src/main/scala/tps/util/MathUtils.scala index ddb9ec56..0ad3ead6 100644 --- a/tps-core/src/main/scala/tps/util/MathUtils.scala +++ b/tps-core/src/main/scala/tps/util/MathUtils.scala @@ -8,13 +8,28 @@ object MathUtils { def max(xs: Seq[Double]): Double = xs.reduceLeft(Math.max) def min(xs: Seq[Double]): Double = xs.reduceLeft(Math.min) - def log2(x: Double) = scala.math.log(x) / scala.math.log(2) + def average(xs: Iterable[Double]): Double = { + xs.sum / xs.size + } - def foldChanges(vs: Seq[Double]): Seq[Double] = { - val baseline = vs.head - vs map (_ / baseline) + def median(xs: Iterable[Double]): Double = { + percentile(xs, 50) } + def percentile(xs: Iterable[Double], p: Int): Double = { + val sorted = xs.toIndexedSeq.sorted + + val rank = (sorted.size * (p / 100.0)).toInt + + if (rank * (100 / p) == sorted.size) { + (sorted(rank - 1) + sorted(rank)) / 2 + } else { + sorted(rank) + } + } + + def log2(x: Double) = scala.math.log(x) / scala.math.log(2) + def combination[A](l: List[A], k: Int): List[List[A]] = { if (k == 0) { List(Nil) diff --git a/tps-core/src/test/scala/tps/ScalaZ3Test.scala b/tps-core/src/test/scala/tps/ScalaZ3Test.scala index 362d8d5e..983f887e 100644 --- a/tps-core/src/test/scala/tps/ScalaZ3Test.scala +++ b/tps-core/src/test/scala/tps/ScalaZ3Test.scala @@ -5,9 +5,9 @@ import org.scalatest.FunSuite import org.scalatest.Matchers class ScalaZ3Test extends FunSuite with Matchers { - test("z3 is accessible and has correct version") { + test("z3 is accessible") { try { - z3.scala.version should startWith ("Z3 4.3") + z3.scala.version should startWith ("Z3") } catch { case e: java.lang.UnsatisfiedLinkError => fail("Cannot access Z3:" + e) diff --git a/tps-core/src/test/scala/tps/util/CollectionUtilsTest.scala b/tps-core/src/test/scala/tps/util/CollectionUtilsTest.scala new file mode 100644 index 00000000..fda6e454 --- /dev/null +++ b/tps-core/src/test/scala/tps/util/CollectionUtilsTest.scala @@ -0,0 +1,13 @@ +package tps.util + +import org.scalatest.FunSuite + +class CollectionUtilsTest extends FunSuite { + test("permute a sequence") { + val permutation = List(1, 0, 3, 2) + val xs = List("a", "b", "c", "d") + + val expected = List("b", "a", "d", "c") + assertResult(expected)(CollectionUtils.permuteSeq(xs, permutation)) + } +} diff --git a/tps-core/src/test/scala/tps/util/MathUtilsTest.scala b/tps-core/src/test/scala/tps/util/MathUtilsTest.scala index 9954b55a..c5afe943 100644 --- a/tps-core/src/test/scala/tps/util/MathUtilsTest.scala +++ b/tps-core/src/test/scala/tps/util/MathUtilsTest.scala @@ -27,4 +27,20 @@ class MathUtilsTest extends FunSuite with Matchers { val c = MathUtils.combination(list, 0) c should equal (List(Nil)) } + + test("median of odd-sized list") { + MathUtils.median(List(1, 2, 3)) should equal (2) + } + + test("median of even-sized list") { + MathUtils.median(List(1, 2, 3, 4)) should equal (2.5) + } + + test("25th percentile of an even-sized list") { + MathUtils.percentile(List(1, 2, 3, 4), 25) should equal (1.5) + } + + test("25th percentile of an odd-sized list") { + MathUtils.percentile(List(1, 2, 3, 4, 5), 25) should equal (2) + } }