diff --git a/ml4h/.idea/.gitignore b/ml4h/.idea/.gitignore
new file mode 100644
index 000000000..26d33521a
--- /dev/null
+++ b/ml4h/.idea/.gitignore
@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml
diff --git a/ml4h/.idea/inspectionProfiles/profiles_settings.xml b/ml4h/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 000000000..105ce2da2
--- /dev/null
+++ b/ml4h/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/ml4h/.idea/misc.xml b/ml4h/.idea/misc.xml
new file mode 100644
index 000000000..d665c63d2
--- /dev/null
+++ b/ml4h/.idea/misc.xml
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/ml4h/.idea/ml4h.iml b/ml4h/.idea/ml4h.iml
new file mode 100644
index 000000000..8b8c39547
--- /dev/null
+++ b/ml4h/.idea/ml4h.iml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/ml4h/.idea/modules.xml b/ml4h/.idea/modules.xml
new file mode 100644
index 000000000..b51e23d9c
--- /dev/null
+++ b/ml4h/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/ml4h/.idea/vcs.xml b/ml4h/.idea/vcs.xml
new file mode 100644
index 000000000..6c0b86358
--- /dev/null
+++ b/ml4h/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/phenotype_labels/disease/cvdidisease.osx b/phenotype_labels/disease/cvdidisease.osx
new file mode 100755
index 000000000..e259c81f9
--- /dev/null
+++ b/phenotype_labels/disease/cvdidisease.osx
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79030f8a4d76e87c5667ab21575b756fdb7237c9d681aa0af998f221da9f9d92
+size 28639504
diff --git a/phenotype_labels/disease/go.mod b/phenotype_labels/disease/go.mod
new file mode 100644
index 000000000..0549f5b41
--- /dev/null
+++ b/phenotype_labels/disease/go.mod
@@ -0,0 +1,52 @@
+module main
+
+go 1.21.0
+
+require (
+ cloud.google.com/go/bigquery v1.54.0
+ github.com/carbocation/pfx v0.0.0-20230108194214-fcea663adae5
+ google.golang.org/api v0.138.0
+)
+
+require (
+ cloud.google.com/go v0.110.7 // indirect
+ cloud.google.com/go/compute v1.23.0 // indirect
+ cloud.google.com/go/compute/metadata v0.2.3 // indirect
+ cloud.google.com/go/iam v1.1.2 // indirect
+ github.com/andybalholm/brotli v1.0.5 // indirect
+ github.com/apache/arrow/go/v12 v12.0.1 // indirect
+ github.com/apache/thrift v0.19.0 // indirect
+ github.com/goccy/go-json v0.10.2 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/golang/protobuf v1.5.3 // indirect
+ github.com/golang/snappy v0.0.4 // indirect
+ github.com/google/flatbuffers v23.5.26+incompatible // indirect
+ github.com/google/go-cmp v0.5.9 // indirect
+ github.com/google/s2a-go v0.1.7 // indirect
+ github.com/google/uuid v1.3.1 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+ github.com/googleapis/gax-go/v2 v2.12.0 // indirect
+ github.com/klauspost/asmfmt v1.3.2 // indirect
+ github.com/klauspost/compress v1.16.7 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
+ github.com/pierrec/lz4/v4 v4.1.18 // indirect
+ github.com/zeebo/xxh3 v1.0.2 // indirect
+ go.opencensus.io v0.24.0 // indirect
+ golang.org/x/crypto v0.13.0 // indirect
+ golang.org/x/mod v0.12.0 // indirect
+ golang.org/x/net v0.15.0 // indirect
+ golang.org/x/oauth2 v0.11.0 // indirect
+ golang.org/x/sync v0.3.0 // indirect
+ golang.org/x/sys v0.12.0 // indirect
+ golang.org/x/text v0.13.0 // indirect
+ golang.org/x/tools v0.12.0 // indirect
+ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+ google.golang.org/appengine v1.6.7 // indirect
+ google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+ google.golang.org/grpc v1.57.0 // indirect
+ google.golang.org/protobuf v1.31.0 // indirect
+)
diff --git a/phenotype_labels/disease/go.sum b/phenotype_labels/disease/go.sum
new file mode 100644
index 000000000..382d0f094
--- /dev/null
+++ b/phenotype_labels/disease/go.sum
@@ -0,0 +1,304 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q=
+cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
+cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o=
+cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
+cloud.google.com/go/bigquery v1.54.0 h1:ify6s7sy+kQuAimRnVTrPUzaeY0+X5GEsKt2C5CiA8w=
+cloud.google.com/go/bigquery v1.54.0/go.mod h1:9Y5I3PN9kQWuid6183JFhOGOW3GcirA5LpsKCUn+2ec=
+cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
+cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
+cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
+cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+cloud.google.com/go/datacatalog v1.16.0 h1:qVeQcw1Cz93/cGu2E7TYUPh8Lz5dn5Ws2siIuQ17Vng=
+cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4=
+cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y=
+cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
+cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4=
+cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
+cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI=
+cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc=
+cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
+cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
+github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
+github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
+github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc=
+github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg=
+github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg=
+github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw=
+github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
+github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
+github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk=
+github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I=
+github.com/carbocation/pfx v0.0.0-20230108194214-fcea663adae5 h1:7A4hRCgyB37Iy4r2XmWnjBz1KgWJaIlcPSgU6rrCIbs=
+github.com/carbocation/pfx v0.0.0-20230108194214-fcea663adae5/go.mod h1:Bc8KYgtyzRcyu6591fpHEnXAavkbE4c2pjbh57Hm2m0=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
+github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM=
+github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg=
+github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
+github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
+github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg=
+github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
+github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
+github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM=
+github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
+github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
+github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
+github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
+github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
+github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
+github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw=
+golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
+golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
+golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
+gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
+google.golang.org/api v0.138.0 h1:K/tVp05MxNVbHShRw9m7e9VJGdagNeTdMzqPH7AUqr0=
+google.golang.org/api v0.138.0/go.mod h1:4xyob8CxC+0GChNBvEUAk8VBKNvYOTWM9T3v3UfRxuY=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g=
+google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
+google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44=
+google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q=
+google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q=
+google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 h1:wukfNtZmZUurLN/atp2hiIeTKn7QJWIQdHzqmsOnAOk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
+google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
+google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/phenotype_labels/disease/main.go b/phenotype_labels/disease/main.go
old mode 100644
new mode 100755
index ef1866389..15b68523f
--- a/phenotype_labels/disease/main.go
+++ b/phenotype_labels/disease/main.go
@@ -25,6 +25,7 @@ var (
)
var materializedDB string
+var useGPData bool
func main() {
defer STDOUT.Flush()
@@ -36,6 +37,7 @@ func main() {
var displayQuery bool
var override bool
var diseaseName string
+
flag.StringVar(&BQ.Project, "project", "", "Google Cloud project you want to use for billing purposes only")
flag.StringVar(&BQ.Database, "database", "", "BigQuery source database name (note: must be formatted as project.database, e.g., broad-ml4cvd.ukbb7089_201904)")
@@ -43,6 +45,7 @@ func main() {
flag.StringVar(&materializedDB, "materialized", "broad-ml4cvd.ukbb7089_201904", "project.database storing materialized view tables")
flag.BoolVar(&displayQuery, "display-query", false, "Display the constructed query and exit?")
flag.BoolVar(&override, "override", false, "Force run, even if this tool thinks your tabfile is inadequate?")
+ flag.BoolVar(&useGPData, "use-gp-data", false, "Use general practitioner data? Note that materialized_gp_dates table must first be created. ")
flag.StringVar(&diseaseName, "disease", "", "If not specified, the tabfile will be parsed and become the disease name.")
flag.Parse()
@@ -62,7 +65,7 @@ func main() {
diseaseName = strings.Join(parts[0:len(parts)-1], ".")
}
}
-
+
log.Println("Processing disease", diseaseName)
missingFields, err := tabs.CheckSensibility()
@@ -80,7 +83,7 @@ func main() {
}
defer BQ.Client.Close()
- query, err := BuildQuery(BQ, tabs, displayQuery)
+ query, err := BuildQuery(BQ, tabs, displayQuery, useGPData)
if err != nil {
log.Fatalln(diseaseName, err)
}
diff --git a/phenotype_labels/disease/materialized_gp_dates.sql b/phenotype_labels/disease/materialized_gp_dates.sql
new file mode 100644
index 000000000..85ed8fe36
--- /dev/null
+++ b/phenotype_labels/disease/materialized_gp_dates.sql
@@ -0,0 +1,33 @@
+WITH read_2_only AS (
+ SELECT gpc.eid sample_id, 41202 FieldID, cv2.icd10_code value, gpc.event_dt vdate,
+ FROM `ukbb-analyses.ukbb7089_202109.gp_clinical` gpc
+ INNER JOIN `ukbb-analyses.ukbb7089_202109.map_read_v2_icd10` cv2 on gpc.read_2 = cv2.read_code
+ WHERE gpc.read_2 is not null and cv2.icd10_code_def=1
+ and cv2.icd10_code not like '%-%' and cv2.icd10_code not like '%,%'
+ and cv2.icd10_code not like '%+%' and cv2.icd10_code not like '% %'
+ and cv2.icd10_code not like '%X'
+), result_read_2 AS (
+ SELECT sample_id, FieldID, value, MIN(vdate) first_date from read_2_only
+ GROUP BY sample_id, FieldID, value
+), read_3_only AS (
+ SELECT gpc.eid sample_id, 41202 FieldID, cv3.icd10_code value, gpc.event_dt vdate,
+ FROM `ukbb-analyses.ukbb7089_202109.gp_clinical` gpc
+ INNER JOIN `ukbb-analyses.ukbb7089_202109.map_read_v3_icd10` cv3 on gpc.read_3 = cv3.read_code
+ WHERE gpc.read_3 is not null
+ and cv3.icd10_code not like '%-%' and cv3.icd10_code not like '%,%'
+ and cv3.icd10_code not like '%+%' and cv3.icd10_code not like '% %'
+ and cv3.icd10_code not like '%X' and cv3.icd10_code not like '%D'
+ and cv3.icd10_code not like '%A'
+ and ((cv3.mapping_status='E' and cv3.refine_flag != 'M') or (cv3.mapping_status='D' and cv3.refine_flag in ('C','P') and cv3.add_code_flag in ('C', 'P', 'M')))
+), result_read_3 AS (
+ SELECT sample_id, FieldID, value, MIN(vdate) first_date from read_3_only
+ GROUP BY sample_id, FieldID, value
+)
+
+SELECT all_results.sample_id, all_results.FieldID, all_results.value, MIN(all_results.first_date) first_date
+FROM (
+ SELECT * FROM result_read_2
+ UNION DISTINCT
+ SELECT * FROM result_read_3
+) all_results
+GROUP BY all_results.sample_id, all_results.FieldID, all_results.value
diff --git a/phenotype_labels/disease/materialized_hesin_dates.sql b/phenotype_labels/disease/materialized_hesin_dates.sql
index 14349eec4..213031f1d 100644
--- a/phenotype_labels/disease/materialized_hesin_dates.sql
+++ b/phenotype_labels/disease/materialized_hesin_dates.sql
@@ -1,70 +1,66 @@
WITH oper4 AS (
- SELECT 41200 FieldID, eid, oper4 code,
+ SELECT 41200 FieldID, oper.eid, oper4 code,
CASE
+ WHEN oper.opdate IS NOT NULL THEN oper.opdate
WHEN h.admidate IS NOT NULL THEN h.admidate
- WHEN h.admidate IS NULL AND h.opdate IS NOT NULL THEN h.opdate
ELSE h.epistart
END vdate
- FROM `broad-ml4cvd.ukbb7089_201904.hesin` h
- WHERE oper4 IS NOT NULL
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.hesin_oper` oper
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.hesin` h ON oper.eid=h.eid AND oper.ins_index = h.ins_index
+ WHERE oper4 IS NOT NULL AND oper.level=1
), diag_icd10 AS (
- SELECT 41202 FieldID, eid, diag_icd10 code,
+ SELECT 41202 FieldID, hd.eid, diag_icd10 code,
CASE
WHEN h.admidate IS NOT NULL THEN h.admidate
- WHEN h.admidate IS NULL AND h.opdate IS NOT NULL THEN h.opdate
ELSE h.epistart
END vdate
- FROM `broad-ml4cvd.ukbb7089_201904.hesin` h
- WHERE diag_icd10 IS NOT NULL
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.hesin_diag` hd
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.hesin` h ON h.eid = hd.eid and h.ins_index = hd.ins_index
+ WHERE diag_icd10 IS NOT NULL and hd.level=1
), diag_icd9 AS (
- SELECT 41203 FieldID, eid, diag_icd9 code,
+ SELECT 41203 FieldID, hd.eid, diag_icd9 code,
CASE
WHEN h.admidate IS NOT NULL THEN h.admidate
- WHEN h.admidate IS NULL AND h.opdate IS NOT NULL THEN h.opdate
ELSE h.epistart
END vdate
- FROM `broad-ml4cvd.ukbb7089_201904.hesin` h
- WHERE diag_icd9 IS NOT NULL
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.hesin_diag` hd
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.hesin` h ON h.eid = hd.eid and h.ins_index = hd.ins_index
+ WHERE diag_icd9 IS NOT NULL and hd.level=1
), oper4secondary AS (
- SELECT 41210 FieldID, h.eid, sec.oper4 code,
+ SELECT 41210 FieldID, sec_oper.eid, sec_oper.oper4 code,
CASE
+ WHEN sec_oper.opdate IS NOT NULL THEN sec_oper.opdate
WHEN h.admidate IS NOT NULL THEN h.admidate
- WHEN h.admidate IS NULL AND h.opdate IS NOT NULL THEN h.opdate
ELSE h.epistart
END vdate
- FROM `broad-ml4cvd.ukbb7089_201904.hesin_oper` sec
- LEFT JOIN `broad-ml4cvd.ukbb7089_201904.hesin` h ON sec.eid=h.eid AND sec.record_id=h.record_id
- WHERE TRUE
- AND sec.oper4 IS NOT NULL
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.hesin_oper` sec_oper
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.hesin` h ON sec_oper.eid=h.eid AND sec_oper.ins_index = h.ins_index
+ WHERE sec_oper.oper4 IS NOT NULL AND sec_oper.level=2
), diag_icd10_secondary AS (
SELECT 41204 FieldID, h.eid, sec.diag_icd10 code,
- CASE
+ CASE
WHEN h.admidate IS NOT NULL THEN h.admidate
- WHEN h.admidate IS NULL AND h.opdate IS NOT NULL THEN h.opdate
ELSE h.epistart
END vdate
- FROM `broad-ml4cvd.ukbb7089_201904.hesin_diag10` sec
- LEFT JOIN `broad-ml4cvd.ukbb7089_201904.hesin` h ON sec.eid=h.eid AND sec.record_id=h.record_id
- WHERE TRUE
- AND sec.diag_icd10 IS NOT NULL
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.hesin_diag` sec
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.hesin` h ON sec.eid=h.eid AND sec.ins_index = h.ins_index
+ WHERE sec.diag_icd10 IS NOT NULL and sec.level=2
), diag_icd9_secondary AS (
SELECT 41205 FieldID, h.eid, sec.diag_icd9 code,
CASE
WHEN h.admidate IS NOT NULL THEN h.admidate
- WHEN h.admidate IS NULL AND h.opdate IS NOT NULL THEN h.opdate
ELSE h.epistart
END vdate
- FROM `broad-ml4cvd.ukbb7089_201904.hesin_diag9` sec
- LEFT JOIN `broad-ml4cvd.ukbb7089_201904.hesin` h ON sec.eid=h.eid AND sec.record_id=h.record_id
- WHERE TRUE
- AND sec.diag_icd9 IS NOT NULL
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.hesin_diag` sec
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.hesin` h ON sec.eid=h.eid AND sec.ins_index = h.ins_index
+ WHERE sec.diag_icd9 IS NOT NULL and sec.level=2
)
SELECT
diagnostics.eid sample_id, diagnostics.FieldID, diagnostics.code value,
- CASE
- WHEN MIN(PARSE_DATE("%E4Y-%m-%d", vdate)) IS NULL THEN MIN(PARSE_DATE("%E4Y-%m-%d", p.value))
- ELSE MIN(PARSE_DATE("%E4Y-%m-%d", vdate))
+ CASE
+ WHEN MIN(PARSE_DATE("%d/%m/%E4Y", vdate)) IS NULL THEN MIN(PARSE_DATE("%E4Y-%m-%d", p.value))
+ ELSE MIN(PARSE_DATE("%d/%m/%E4Y", vdate))
END first_date
FROM (
SELECT * FROM oper4
@@ -79,7 +75,7 @@ FROM (
UNION DISTINCT
SELECT * FROM diag_icd9_secondary
) diagnostics
-JOIN `broad-ml4cvd.ukbb7089_201904.phenotype` p ON p.sample_id = diagnostics.eid AND p.array_idx=0 AND p.instance=0 AND p.FieldID=53
-GROUP BY diagnostics.eid, diagnostics.FieldID, diagnostics.code
+JOIN `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` p ON p.sample_id = diagnostics.eid AND p.array_idx=0 AND p.instance=0 AND p.FieldID=53
+GROUP BY diagnostics.eid, diagnostics.FieldID, diagnostics.code, diagnostics.vdate
ORDER BY first_date ASC
-;
\ No newline at end of file
+
diff --git a/phenotype_labels/disease/materialized_special_dates.sql b/phenotype_labels/disease/materialized_special_dates.sql
index 2aa21186d..b2091eb23 100644
--- a/phenotype_labels/disease/materialized_special_dates.sql
+++ b/phenotype_labels/disease/materialized_special_dates.sql
@@ -5,9 +5,9 @@ WITH dated_fields AS (
WHEN cod.meaning LIKE ('%unknown%') THEN SAFE.PARSE_DATE("%E4Y-%m-%d", denroll.value)
ELSE SAFE.PARSE_DATE("%E4Y-%m-%d", d.value)
END vdate
- FROM `broad-ml4cvd.ukbb7089_201904.phenotype` p
- JOIN `broad-ml4cvd.ukbb7089_201904.phenotype` denroll ON denroll.FieldID=53 AND denroll.sample_id=p.sample_id AND denroll.instance = 0 AND denroll.array_idx = 0
- JOIN `broad-ml4cvd.ukbb7089_201904.phenotype` d ON d.sample_id=p.sample_id AND d.instance = p.instance AND d.array_idx = p.array_idx
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` p
+ JOIN `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` denroll ON denroll.FieldID=53 AND denroll.sample_id=p.sample_id AND denroll.instance = 0 AND denroll.array_idx = 0
+ JOIN `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` d ON d.sample_id=p.sample_id AND d.instance = p.instance AND d.array_idx = p.array_idx
AND (
FALSE
OR (p.FieldID=42013 AND d.FieldID=42012)
@@ -16,25 +16,40 @@ WITH dated_fields AS (
OR (p.FieldID=42007 AND d.FieldID=42006)
OR (p.FieldID=42001 AND d.FieldID=42000)
)
- LEFT JOIN `broad-ml4cvd.ukbb7089_201904.coding` cod ON cod.coding_file_id = d.coding_file_id AND cod.coding = d.value
-),
-dated_fields_fractional AS (
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.coding` cod ON cod.coding_file_id = d.coding_file_id AND cod.coding = d.value
+),dated_death_cause_fields AS (
+ SELECT p.FieldID, p.sample_id eid, p.value code, cod.meaning,
+ CASE
+ WHEN SAFE.PARSE_DATE("%E4Y-%m-%d", d.value) IS NULL THEN SAFE.PARSE_DATE("%E4Y-%m-%d", denroll.value)
+ WHEN cod.meaning LIKE ('%unknown%') THEN SAFE.PARSE_DATE("%E4Y-%m-%d", denroll.value)
+ ELSE SAFE.PARSE_DATE("%E4Y-%m-%d", d.value)
+ END vdate
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` p
+ JOIN `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` denroll ON denroll.FieldID=53 AND denroll.sample_id=p.sample_id AND denroll.instance = 0 AND denroll.array_idx = 0
+ JOIN `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` d ON d.sample_id=p.sample_id AND d.instance = p.instance
+ AND (
+ FALSE
+ OR (p.FieldID=40001 AND d.FieldID=40000)
+ OR (p.FieldID=40002 AND d.FieldID=40000)
+ )
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.coding` cod ON cod.coding_file_id = d.coding_file_id AND cod.coding = d.value
+),dated_fields_fractional AS (
SELECT p.FieldID, p.sample_id eid, p.value code, cod.meaning,
CASE
WHEN SAFE.PARSE_DATE("%Y", d.value) IS NULL THEN SAFE.PARSE_DATE("%E4Y-%m-%d", denroll.value)
WHEN cod.meaning LIKE ('%unknown%') THEN SAFE.PARSE_DATE("%E4Y-%m-%d", denroll.value)
ELSE SAFE.PARSE_DATE("%Y", d.value)
END vdate
- FROM `broad-ml4cvd.ukbb7089_201904.phenotype` p
- JOIN `broad-ml4cvd.ukbb7089_201904.phenotype` denroll ON denroll.FieldID=53 AND denroll.sample_id=p.sample_id AND denroll.instance = 0 AND denroll.array_idx = 0
- JOIN `broad-ml4cvd.ukbb7089_201904.phenotype` d ON d.sample_id=p.sample_id AND d.instance = p.instance AND d.array_idx = p.array_idx
+ FROM `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` p
+ JOIN `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` denroll ON denroll.FieldID=53 AND denroll.sample_id=p.sample_id AND denroll.instance = 0 AND denroll.array_idx = 0
+ JOIN `broad-ml4cvd.ukbb7089_2024_01_20.phenotype` d ON d.sample_id=p.sample_id AND d.instance = p.instance AND d.array_idx = p.array_idx
AND (
FALSE
OR (p.FieldID=20004 AND d.FieldID=20010)
OR (p.FieldID=20002 AND d.FieldID=20008)
OR (p.FieldID=20001 AND d.FieldID=20006)
)
- LEFT JOIN `broad-ml4cvd.ukbb7089_201904.coding` cod ON cod.coding_file_id = d.coding_file_id AND cod.coding = d.value
+ LEFT JOIN `broad-ml4cvd.ukbb7089_2024_01_20.coding` cod ON cod.coding_file_id = d.coding_file_id AND cod.coding = d.value
)
SELECT
@@ -42,6 +57,8 @@ SELECT
FROM (
SELECT * FROM dated_fields
UNION DISTINCT
+ SELECT * FROM dated_death_cause_fields
+ UNION DISTINCT
SELECT * FROM dated_fields_fractional
) diagnostics
WHERE TRUE
diff --git a/phenotype_labels/disease/process_tab_files.sh b/phenotype_labels/disease/process_tab_files.sh
new file mode 100755
index 000000000..e91ac6887
--- /dev/null
+++ b/phenotype_labels/disease/process_tab_files.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+
+TAB_FILES_PATH="/Users/asmirnov/Desktop/ml4h/ukbb/tab_files/batch-13"
+PROJECT="broad-ml4cvd"
+DATABASE="ukbb7089_2024_01_20"
+GO_CODE_PATH="/Users/asmirnov/Desktop/repos/ml4h/phenotype_labels/disease"
+TEMP_OUTPUT_DIR="/Users/asmirnov/Desktop/ml4h/ukbb/temp_output/"
+
+# Check if use-gp-data positional argument is provided
+if [ -z "$1" ]; then
+ echo "Error: Please specify whether to use general practitioner (GP) data for generating disease table."
+ echo "Usage: $0 "
+ exit 1
+fi
+
+# Check if the use-gp-data argument is either "true" or "false"
+if [ "$1" != "true" ] && [ "$1" != "false" ]; then
+ echo "Error: Invalid argument. Only 'true' or 'false' are allowed."
+ echo "Usage: $0 "
+ exit 1
+fi
+
+USE_GP_DATA=$1
+if [ ${USE_GP_DATA} == "false" ]; then
+ DATABASE_NAME=${DATABASE}.disease
+else
+ DATABASE_NAME=${DATABASE}.disease_gp
+fi
+
+mkdir -p ${TEMP_OUTPUT_DIR}
+
+if [ ! -d ${TAB_FILES_PATH} ]; then
+ echo "Error: ${TAB_FILES_PATH} is not a directory!"
+ exit 1
+fi
+
+cd ${GO_CODE_PATH}
+for file in ${TAB_FILES_PATH}/*.tab; do
+ # Check if it's a regular file or a directory
+ file_name="$(basename "$file")"
+ go run . -database "${PROJECT}.${DATABASE}" -materialized "${PROJECT}.${DATABASE}" -project "broad-ml4cvd" -override=true -use-gp-data=${USE_GP_DATA} -tabfile ${file} > ${TEMP_OUTPUT_DIR}/${file_name}.tsv
+ bq load --source_format=CSV --field_delimiter='\t' --autodetect ${DATABASE_NAME} ${TEMP_OUTPUT_DIR}/${file_name}.tsv
+ rm ${TEMP_OUTPUT_DIR}/${file_name}.tsv
+done
diff --git a/phenotype_labels/disease/result.go b/phenotype_labels/disease/result.go
index ef4efe04b..2fa5b62f9 100644
--- a/phenotype_labels/disease/result.go
+++ b/phenotype_labels/disease/result.go
@@ -18,6 +18,7 @@ type Result struct {
HasDisease bigquery.NullInt64 `bigquery:"has_disease"`
IncidentDisease bigquery.NullInt64 `bigquery:"incident_disease"`
PrevalentDisease bigquery.NullInt64 `bigquery:"prevalent_disease"`
+ MetExclusion bigquery.NullInt64 `bigquery:"met_exclusion"`
PhenotypeDateCensor bigquery.NullDate `bigquery:"date_censor"`
RoughPhenotypeAgeCensor bigquery.NullFloat64 `bigquery:"age_censor"` // Note: just uses days/365. Don't use.
BirthDate bigquery.NullDate `bigquery:"birthdate"`
@@ -93,7 +94,7 @@ func ExecuteQuery(BQ *WrappedBigQuery, query *bigquery.Query, diseaseName string
}
todayDate := time.Now().Format("2006-01-02")
missing := strings.Join(missingFields, ",")
- fmt.Fprintf(STDOUT, "disease\tsample_id\thas_disease\tincident_disease\tprevalent_disease\tcensor_date\tcensor_age\tbirthdate\tenroll_date\tenroll_age\thas_died\tdeath_censor_date\tdeath_censor_age\tcensor_computed_date\tcensor_missing_fields\tcomputed_date\tmissing_fields\n")
+ fmt.Fprintf(STDOUT, "disease\tsample_id\thas_disease\tincident_disease\tprevalent_disease\tmet_exclusion\tcensor_date\tcensor_age\tbirthdate\tenroll_date\tenroll_age\thas_died\tdeath_censor_date\tdeath_censor_age\tcensor_computed_date\tcensor_missing_fields\tcomputed_date\tmissing_fields\n")
for {
var r Result
err := itr.Next(&r)
@@ -130,8 +131,8 @@ func ExecuteQuery(BQ *WrappedBigQuery, query *bigquery.Query, diseaseName string
r.DeathDate = r.EnrollDate
}
- fmt.Fprintf(STDOUT, "%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
- diseaseName, r.SampleID, NA(r.HasDisease), NA(r.IncidentDisease), NA(r.PrevalentDisease), NA(r.PhenotypeDateCensor), NA(censoredPhenoAge), NA(r.BirthDate), NA(r.EnrollDate), NA(r.EnrollAge), NA(r.HasDied), NA(r.DeathDate), NA(censoredDeathAge), NA(r.ComputedDate), NA(r.MissingFields), todayDate, missing)
+ fmt.Fprintf(STDOUT, "%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
+ diseaseName, r.SampleID, NA(r.HasDisease), NA(r.IncidentDisease), NA(r.PrevalentDisease), NA(r.MetExclusion), NA(r.PhenotypeDateCensor), NA(censoredPhenoAge), NA(r.BirthDate), NA(r.EnrollDate), NA(r.EnrollAge), NA(r.HasDied), NA(r.DeathDate), NA(censoredDeathAge), NA(r.ComputedDate), NA(r.MissingFields), todayDate, missing)
}
return nil
@@ -164,7 +165,7 @@ func NA(input interface{}) interface{} {
return input
}
-func BuildQuery(BQ *WrappedBigQuery, tabs *TabFile, displayQuery bool) (*bigquery.Query, error) {
+func BuildQuery(BQ *WrappedBigQuery, tabs *TabFile, displayQuery bool, useGPData bool) (*bigquery.Query, error) {
query := rawQuery(BQ)
params := []bigquery.QueryParameter{}
@@ -188,6 +189,14 @@ func BuildQuery(BQ *WrappedBigQuery, tabs *TabFile, displayQuery bool) (*bigquer
}
}
+
+ // Debugging
+ // for _, v := range includedValues {
+ // fmt.Printf("FieldID: %v, Values: %v\n", v.FieldID, v.FormattedValues())
+ // }
+ // fmt.Println("includePart:", includePart)
+
+
excludePart := ""
exludedValues := tabs.AllExcluded()
if len(exludedValues) > 0 {
@@ -200,6 +209,13 @@ func BuildQuery(BQ *WrappedBigQuery, tabs *TabFile, displayQuery bool) (*bigquer
}
}
+
+ // Debugging
+ // for _, v := range exludedValues {
+ // fmt.Printf("FieldID: %v, Values: %v\n", v.FieldID, v.FormattedValues())
+ // }
+ // fmt.Println("excludePart:", excludePart)
+
// Assemble all the parts
query = fmt.Sprintf(query, standardPart, includePart, excludePart)
@@ -229,6 +245,10 @@ func BuildQuery(BQ *WrappedBigQuery, tabs *TabFile, displayQuery bool) (*bigquer
// TODO: Resolve age_censor vs enroll_age. Choose one or the other (likely the
// latter, so you end up with enroll_age, censor_age, death_censor_age).
func rawQuery(BQ *WrappedBigQuery) string {
+ var useGPDataString = ""
+ if useGPData {
+ useGPDataString = fmt.Sprintf("\n\t\t\tUNION DISTINCT\n\t\t\tSELECT * FROM `%s.materialized_gp_dates`", materializedDB)
+ }
return fmt.Sprintf(`
WITH undated_fields AS (
SELECT p.sample_id, p.FieldID, p.value, MIN(SAFE.PARSE_DATE("%%%%E4Y-%%%%m-%%%%d", denroll.value)) first_date
@@ -271,7 +291,7 @@ func rawQuery(BQ *WrappedBigQuery) string {
LEFT OUTER JOIN (
SELECT * FROM `+"`%s.materialized_hesin_dates`"+`
UNION DISTINCT
- SELECT * FROM `+"`%s.materialized_special_dates`"+`
+ SELECT * FROM `+"`%s.materialized_special_dates`%s"+`
UNION DISTINCT
SELECT * FROM undated_fields
) hd ON c.sample_id=hd.sample_id
@@ -279,10 +299,9 @@ func rawQuery(BQ *WrappedBigQuery) string {
FALSE
%%s
)
-
GROUP BY sample_id
)
- ),excluded_only AS (
+ ), excluded_only AS (
SELECT sample_id, has_disease, incident_disease, prevalent_disease, date_censor, DATE_DIFF(date_censor,birthdate, DAY)/365.0 age_censor,
birthdate, enroll_date, enroll_age, death_date, death_age, computed_date, missing_fields
FROM (
@@ -316,7 +335,7 @@ func rawQuery(BQ *WrappedBigQuery) string {
LEFT OUTER JOIN (
SELECT * FROM `+"`%s.materialized_hesin_dates`"+`
UNION DISTINCT
- SELECT * FROM `+"`%s.materialized_special_dates`"+`
+ SELECT * FROM `+"`%s.materialized_special_dates`%s"+`
UNION DISTINCT
SELECT * FROM undated_fields
) excl ON c.sample_id=excl.sample_id
@@ -336,9 +355,9 @@ func rawQuery(BQ *WrappedBigQuery) string {
WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(io.date_censor,eo.date_censor, DAY) > 0 THEN NULL
-- Exclusion occurred after disease onset; we'll allow it:
WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN io.has_disease
- -- Met exclusion but no inclusion
- WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN NULL
- -- Didn't meet exclusion or inclusion means we censor at the date given by UKBB:
+ -- Met exclusion but no inclusion
+ WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN 0
+ -- Didn't meet exclusion or inclusion means we censor at the date given by UKBB:
WHEN io.has_disease IS NULL THEN 0
ELSE io.has_disease
END has_disease,
@@ -348,9 +367,9 @@ func rawQuery(BQ *WrappedBigQuery) string {
-- Exclusion occurred after enrollment and prior to disease onset; we will exclude:
WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(io.date_censor,eo.date_censor, DAY) > 0 THEN NULL
-- Exclusion occurred after disease onset; we'll allow it:
- WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN io.incident_disease
- -- Met exclusion but no inclusion
- WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN NULL
+ WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN io.incident_disease
+ -- Met exclusion but no inclusion
+ WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN 0
-- Didn't meet exclusion or inclusion means we censor at the date given by UKBB:
WHEN io.has_disease IS NULL THEN 0
ELSE io.incident_disease
@@ -361,22 +380,35 @@ func rawQuery(BQ *WrappedBigQuery) string {
-- Exclusion occurred after enrollment and prior to disease onset; we will exclude:
WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(io.date_censor,eo.date_censor, DAY) > 0 THEN NULL
-- Exclusion occurred after disease onset; we'll allow it:
- WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN io.prevalent_disease
- -- Met exclusion but no inclusion
- WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN NULL
+ WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN io.prevalent_disease
+ -- Met exclusion but no inclusion
+ WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN 0
-- Didn't meet exclusion or inclusion means we censor at the date given by UKBB:
WHEN io.has_disease IS NULL THEN 0
ELSE io.prevalent_disease
END prevalent_disease,
+ CASE
+ -- Enrollment occurred after exclusion:
+ WHEN eo.has_disease = 1 AND SAFE.DATE_DIFF(c.enroll_date, eo.date_censor, DAY) > 0 THEN 1
+ -- Exclusion occurred after enrollment and prior to disease onset; we will exclude:
+ WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(io.date_censor,eo.date_censor, DAY) > 0 THEN 1
+ -- Exclusion occurred after disease onset; we'll allow it:
+ WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN 0
+ -- Met exclusion but no inclusion
+ WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN 1
+ -- Didn't meet exclusion or inclusion means we censor at the date given by UKBB:
+ WHEN io.has_disease IS NULL THEN 0
+ ELSE 0
+ END met_exclusion,
CASE
-- Enrollment occurred after exclusion:
WHEN eo.has_disease = 1 AND SAFE.DATE_DIFF(c.enroll_date, eo.date_censor, DAY) > 0 THEN eo.date_censor
-- Exclusion occurred after enrollment and prior to disease onset; we will exclude:
WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(io.date_censor,eo.date_censor, DAY) > 0 THEN eo.date_censor
-- Exclusion occurred after disease onset; we'll allow it:
- WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN io.date_censor
- -- Met exclusion but no inclusion
- WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN eo.date_censor
+ WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN io.date_censor
+ -- Met exclusion but no inclusion
+ WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN eo.date_censor
-- Didn't meet exclusion or inclusion means we censor at the date given by UKBB:
WHEN io.has_disease IS NULL THEN c.phenotype_censor_date
ELSE io.date_censor
@@ -387,9 +419,9 @@ func rawQuery(BQ *WrappedBigQuery) string {
-- Exclusion occurred after enrollment and prior to disease onset; we will exclude:
WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(io.date_censor,eo.date_censor, DAY) > 0 THEN DATE_DIFF(eo.date_censor,c.birthdate, DAY)/365.0
-- Exclusion occurred after disease onset; we'll allow it:
- WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN DATE_DIFF(io.date_censor,c.birthdate, DAY)/365.0
- -- Met exclusion but no inclusion
- WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN DATE_DIFF(eo.date_censor,c.birthdate, DAY)/365.0
+ WHEN eo.has_disease = 1 AND io.has_disease = 1 AND SAFE.DATE_DIFF(eo.date_censor,io.date_censor, DAY) > 0 THEN DATE_DIFF(io.date_censor,c.birthdate, DAY)/365.0
+ -- Met exclusion but no inclusion
+ WHEN eo.has_disease = 1 AND (io.has_disease = 0 OR io.has_disease IS NULL) THEN DATE_DIFF(eo.date_censor,c.birthdate, DAY)/365.0
-- Didn't meet exclusion or inclusion means we censor at the date given by UKBB:
WHEN io.has_disease IS NULL THEN c.phenotype_censor_age
ELSE DATE_DIFF(io.date_censor,c.birthdate, DAY)/365.0
@@ -407,5 +439,5 @@ func rawQuery(BQ *WrappedBigQuery) string {
LEFT JOIN excluded_only eo ON eo.sample_id=c.sample_id
ORDER BY has_disease DESC, incident_disease DESC, age_censor ASC
`,
- BQ.Database, BQ.Database, BQ.Database, materializedDB, materializedDB, BQ.Database, materializedDB, materializedDB, BQ.Database)
+ BQ.Database, BQ.Database, BQ.Database, materializedDB, materializedDB, useGPDataString, BQ.Database, materializedDB, materializedDB, useGPDataString, BQ.Database)
}
diff --git a/phenotype_labels/disease/special_fields.go b/phenotype_labels/disease/special_fields.go
index d29dbf976..2e9784d1c 100644
--- a/phenotype_labels/disease/special_fields.go
+++ b/phenotype_labels/disease/special_fields.go
@@ -11,8 +11,6 @@ var (
41210: struct{}{},
41202: struct{}{},
41204: struct{}{},
- 40001: struct{}{},
- 40002: struct{}{},
41200: struct{}{},
41203: struct{}{},
41205: struct{}{},
@@ -24,6 +22,8 @@ var (
42009: struct{}{},
42007: struct{}{},
42001: struct{}{},
+ 40001: struct{}{},
+ 40002: struct{}{},
20004: struct{}{},
20002: struct{}{},
20001: struct{}{},
diff --git a/phenotype_labels/disease/tabfile.go b/phenotype_labels/disease/tabfile.go
index 8b0649d75..de6a0da1a 100644
--- a/phenotype_labels/disease/tabfile.go
+++ b/phenotype_labels/disease/tabfile.go
@@ -22,13 +22,9 @@ type TabEntry struct {
func (t TabEntry) FormattedValues() []string {
out := make([]string, 0, len(t.Values))
- // HESIN is special-cased to exclude "." (So K41.2 becomes K412)
- if IsHesin(t.FieldID) {
- for _, v := range t.Values {
- out = append(out, strings.Replace(v, ".", "", -1))
- }
- } else {
- out = append(out, t.Values...)
+ // ICD codes are formatted to exclude "." (So K41.2 becomes K412)
+ for _, v := range t.Values {
+ out = append(out, strings.Replace(v, ".", "", -1))
}
// Every field will get leading and trailing spaces trimmed
diff --git a/scripts/vm_launch/launch_dl_instance.sh b/scripts/vm_launch/launch_dl_instance.sh
index 53758670f..1b09e43bb 100755
--- a/scripts/vm_launch/launch_dl_instance.sh
+++ b/scripts/vm_launch/launch_dl_instance.sh
@@ -2,9 +2,9 @@
NAME=${1:-sam-p4}
shift 1
-INSTANCE_TYPE=${1:-n1-standard-4}
+INSTANCE_TYPE=${1:-n1-highmem-32}
shift 1
-DISK_SIZE=${1:-100GB}
+DISK_SIZE=${1:-200GB}
shift 1
ACCEL=${1:-nvidia-tesla-t4}
shift 1