diff --git a/go.mod b/go.mod index ebfc2b7..4239ae8 100644 --- a/go.mod +++ b/go.mod @@ -5,27 +5,21 @@ go 1.23.1 require ( github.com/caffix/stringset v0.2.0 github.com/glebarez/sqlite v1.11.0 - github.com/owasp-amass/open-asset-model v0.9.1 + github.com/owasp-amass/open-asset-model v0.10.0-dev github.com/rubenv/sql-migrate v1.7.0 github.com/stretchr/testify v1.9.0 gorm.io/datatypes v1.2.4 gorm.io/driver/postgres v1.5.9 - gorm.io/driver/sqlite v1.5.4 gorm.io/gorm v1.25.12 ) require ( filippo.io/edwards25519 v1.1.0 // indirect - github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/badger v1.6.2 // indirect - github.com/dgraph-io/ristretto v1.0.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/glebarez/go-sqlite v1.22.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-sql-driver/mysql v1.8.1 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -34,21 +28,18 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.19 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/stretchr/objx v0.5.2 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect - golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.5.7 // indirect + gorm.io/driver/sqlite v1.5.4 // indirect modernc.org/libc v1.61.0 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect diff --git a/go.sum b/go.sum index 6b4531d..33d3d25 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,12 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/caffix/stringset v0.1.2 h1:AnBiZ5dH8AqOtDsUPdFt7ZzHk5RqmGixmfZFlxzZh4U= -github.com/caffix/stringset v0.1.2/go.mod h1:eWeJ1l/1Tc3SO5eybwwMIltkoPNkej2y5d4sHQlHOxw= github.com/caffix/stringset v0.2.0 h1:kN6xnvL8jzx2YhQNOYr6A6hFzUK+iikt1JtJ2MS2LC8= github.com/caffix/stringset v0.2.0/go.mod h1:8PZ6GIPpMP5+r5hr790/05w3v9xI+gXRxRzJCZL57lQ= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= -github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v1.0.0 h1:SYG07bONKMlFDUYu5pEu3DGAh8c2OFNzKm6G9J4Si84= -github.com/dgraph-io/ristretto v1.0.0/go.mod h1:jTi2FiYEhQ1NsMmA7DeBykizjOuY88NhKBkepyu1jPc= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= @@ -45,17 +20,10 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -68,32 +36,22 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= github.com/microsoft/go-mssqldb v0.17.0/go.mod h1:OkoNGhGEs8EZqchVTtochlXruEhEOaO4S0d2sB5aeGQ= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/owasp-amass/open-asset-model v0.9.1 h1:65f+hd1slwKsHqKhyWCf1pqsO42nBBVeHFdBT8agowY= -github.com/owasp-amass/open-asset-model v0.9.1/go.mod h1:DOX+SiD6PZBroSMnsILAmpf0SHi6TVpqjV4uNfBeg7g= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/owasp-amass/open-asset-model v0.10.0-dev h1:P12m9kIMxqiL3RFjCF6RM6ZpMq5t7cimmBfxCy+BAfs= +github.com/owasp-amass/open-asset-model v0.10.0-dev/go.mod h1:DOX+SiD6PZBroSMnsILAmpf0SHi6TVpqjV4uNfBeg7g= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= @@ -104,62 +62,34 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/datatypes v1.2.3 h1:95ucr9ip9dZMPhB3Tc9zbcoAi62hxYAgHicu7SLjK4g= -gorm.io/datatypes v1.2.3/go.mod h1:f4BsLcFAX67szSv8svwLRjklArSHAvHLeE3pXAS5DZI= gorm.io/datatypes v1.2.4 h1:uZmGAcK/QZ0uyfCuVg0VQY1ZmV9h1fuG0tMwKByO1z4= gorm.io/datatypes v1.2.4/go.mod h1:f4BsLcFAX67szSv8svwLRjklArSHAvHLeE3pXAS5DZI= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= diff --git a/migrations/postgres/001_schema_init.sql b/migrations/postgres/001_schema_init.sql index e141ad6..356138b 100644 --- a/migrations/postgres/001_schema_init.sql +++ b/migrations/postgres/001_schema_init.sql @@ -12,83 +12,83 @@ CREATE TABLE IF NOT EXISTS entities( CREATE INDEX idx_entities_last_seen ON entities (last_seen); CREATE INDEX idx_entities_etype ON entities (etype); -CREATE TABLE IF NOT EXISTS entity_properties( - property_id INT GENERATED ALWAYS AS IDENTITY, +CREATE TABLE IF NOT EXISTS entity_tags( + tag_id INT GENERATED ALWAYS AS IDENTITY, created_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, last_seen TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, - ptype VARCHAR(255), + ttype VARCHAR(255), content JSONB, entity_id INT, - PRIMARY KEY(property_id), - CONSTRAINT fk_entity_properties_entities + PRIMARY KEY(tag_id), + CONSTRAINT fk_entity_tags_entities FOREIGN KEY(entity_id) REFERENCES entities(entity_id) ON DELETE CASCADE ); -CREATE INDEX idx_entprop_last_seen ON entity_properties (last_seen); -CREATE INDEX idx_entprop_ptype ON entity_properties (ptype); -CREATE INDEX idx_entprop_entity_id ON entity_properties (entity_id); +CREATE INDEX idx_enttag_last_seen ON entity_tags (last_seen); +CREATE INDEX idx_enttag_ttype ON entity_tags (ttype); +CREATE INDEX idx_enttag_entity_id ON entity_tags (entity_id); -CREATE TABLE IF NOT EXISTS relations( - relation_id INT GENERATED ALWAYS AS IDENTITY, +CREATE TABLE IF NOT EXISTS edges( + edge_id INT GENERATED ALWAYS AS IDENTITY, created_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, last_seen TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, - rtype VARCHAR(255), + etype VARCHAR(255), content JSONB, from_entity_id INT, to_entity_id INT, - PRIMARY KEY(relation_id), - CONSTRAINT fk_relations_entities_from + PRIMARY KEY(edge_id), + CONSTRAINT fk_edges_entities_from FOREIGN KEY(from_entity_id) REFERENCES entities(entity_id) ON DELETE CASCADE, - CONSTRAINT fk_relations_entities_to + CONSTRAINT fk_edges_entities_to FOREIGN KEY(to_entity_id) REFERENCES entities(entity_id) ON DELETE CASCADE ); -CREATE INDEX idx_rel_last_seen ON relations (last_seen); -CREATE INDEX idx_rel_rtype ON relations (rtype); -CREATE INDEX idx_rel_from_entity_id ON relations (from_entity_id); -CREATE INDEX idx_rel_to_entity_id ON relations (to_entity_id); +CREATE INDEX idx_edge_last_seen ON edges (last_seen); +CREATE INDEX idx_edge_etype ON edges (etype); +CREATE INDEX idx_edge_from_entity_id ON edges (from_entity_id); +CREATE INDEX idx_edge_to_entity_id ON edges (to_entity_id); -CREATE TABLE IF NOT EXISTS relation_properties( - property_id INT GENERATED ALWAYS AS IDENTITY, +CREATE TABLE IF NOT EXISTS edge_tags( + tag_id INT GENERATED ALWAYS AS IDENTITY, created_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, last_seen TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, - ptype VARCHAR(255), + ttype VARCHAR(255), content JSONB, - relation_id INT, - PRIMARY KEY(property_id), - CONSTRAINT fk_relation_properties_relations - FOREIGN KEY(relation_id) - REFERENCES relations(relation_id) + edge_id INT, + PRIMARY KEY(tag_id), + CONSTRAINT fk_edge_tags_edges + FOREIGN KEY(edge_id) + REFERENCES edges(edge_id) ON DELETE CASCADE ); -CREATE INDEX idx_relprop_last_seen ON relation_properties (last_seen); -CREATE INDEX idx_relprop_ptype ON relation_properties (ptype); -CREATE INDEX idx_relprop_relation_id ON relation_properties (relation_id); +CREATE INDEX idx_edgetag_last_seen ON edge_tags (last_seen); +CREATE INDEX idx_edgetag_ttype ON edge_tags (ttype); +CREATE INDEX idx_edgetag_edge_id ON edge_tags (edge_id); -- +migrate Down -DROP INDEX IF EXISTS idx_relprop_relation_id; -DROP INDEX IF EXISTS idx_relprop_ptype; -DROP INDEX IF EXISTS idx_relprop_last_seen; -DROP TABLE relation_properties; +DROP INDEX IF EXISTS idx_edgetag_edge_id; +DROP INDEX IF EXISTS idx_edgetag_ttype; +DROP INDEX IF EXISTS idx_edgetag_last_seen; +DROP TABLE edge_tags; -DROP INDEX IF EXISTS idx_rel_to_entity_id; -DROP INDEX IF EXISTS idx_rel_from_entity_id; -DROP INDEX IF EXISTS idx_rel_rtype; -DROP INDEX IF EXISTS idx_rel_last_seen; -DROP TABLE relations; +DROP INDEX IF EXISTS idx_edge_to_entity_id; +DROP INDEX IF EXISTS idx_edge_from_entity_id; +DROP INDEX IF EXISTS idx_edge_etype; +DROP INDEX IF EXISTS idx_edge_last_seen; +DROP TABLE edges; -DROP INDEX IF EXISTS idx_entprop_entity_id; -DROP INDEX IF EXISTS idx_entprop_ptype; -DROP INDEX IF EXISTS idx_entprop_last_seen; -DROP TABLE entity_properties; +DROP INDEX IF EXISTS idx_enttag_entity_id; +DROP INDEX IF EXISTS idx_enttag_ttype; +DROP INDEX IF EXISTS idx_enttag_last_seen; +DROP TABLE entity_tags; DROP INDEX IF EXISTS idx_entities_etype; DROP INDEX IF EXISTS idx_entities_last_seen; diff --git a/migrations/sqlite3/001_schema_init.sql b/migrations/sqlite3/001_schema_init.sql index 66ab2d3..1b671aa 100644 --- a/migrations/sqlite3/001_schema_init.sql +++ b/migrations/sqlite3/001_schema_init.sql @@ -14,11 +14,11 @@ CREATE TABLE IF NOT EXISTS entities( CREATE INDEX idx_entities_last_seen ON entities (last_seen); CREATE INDEX idx_entities_etype ON entities (etype); -CREATE TABLE IF NOT EXISTS entity_properties( - property_id INTEGER PRIMARY KEY, +CREATE TABLE IF NOT EXISTS entity_tags( + tag_id INTEGER PRIMARY KEY, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, - ptype TEXT, + ttype TEXT, content TEXT, entity_id INTEGER, FOREIGN KEY(entity_id) @@ -26,15 +26,15 @@ CREATE TABLE IF NOT EXISTS entity_properties( ON DELETE CASCADE ); -CREATE INDEX idx_entprop_last_seen ON entity_properties (last_seen); -CREATE INDEX idx_entprop_ptype ON entity_properties (ptype); -CREATE INDEX idx_entprop_entity_id ON entity_properties (entity_id); +CREATE INDEX idx_enttag_last_seen ON entity_tags (last_seen); +CREATE INDEX idx_enttag_ttype ON entity_tags (ttype); +CREATE INDEX idx_enttag_entity_id ON entity_tags (entity_id); -CREATE TABLE IF NOT EXISTS relations( - relation_id INTEGER PRIMARY KEY, +CREATE TABLE IF NOT EXISTS edges( + edge_id INTEGER PRIMARY KEY, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, - rtype TEXT, + etype TEXT, content TEXT, from_entity_id INTEGER, to_entity_id INTEGER, @@ -46,44 +46,44 @@ CREATE TABLE IF NOT EXISTS relations( ON DELETE CASCADE ); -CREATE INDEX idx_rel_last_seen ON relations (last_seen); -CREATE INDEX idx_rel_rtype ON relations (rtype); -CREATE INDEX idx_rel_from_entity_id ON relations (from_entity_id); -CREATE INDEX idx_rel_to_entity_id ON relations (to_entity_id); +CREATE INDEX idx_edge_last_seen ON edges (last_seen); +CREATE INDEX idx_edge_etype ON edges (etype); +CREATE INDEX idx_edge_from_entity_id ON edges (from_entity_id); +CREATE INDEX idx_edge_to_entity_id ON edges (to_entity_id); -CREATE TABLE IF NOT EXISTS relation_properties( - property_id INTEGER PRIMARY KEY, +CREATE TABLE IF NOT EXISTS edge_tags( + tag_id INTEGER PRIMARY KEY, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, - ptype TEXT, + ttype TEXT, content TEXT, - relation_id INTEGER, - FOREIGN KEY(relation_id) - REFERENCES relations(relation_id) + edge_id INTEGER, + FOREIGN KEY(edge_id) + REFERENCES edges(edge_id) ON DELETE CASCADE ); -CREATE INDEX idx_relprop_last_seen ON relation_properties (last_seen); -CREATE INDEX idx_relprop_ptype ON relation_properties (ptype); -CREATE INDEX idx_relprop_relation_id ON relation_properties (relation_id); +CREATE INDEX idx_edgetag_last_seen ON edge_tags (last_seen); +CREATE INDEX idx_edgetag_ttype ON edge_tags (ttype); +CREATE INDEX idx_edgetag_edge_id ON edge_tags (edge_id); -- +migrate Down -DROP INDEX IF EXISTS idx_relprop_relation_id; -DROP INDEX IF EXISTS idx_relprop_ptype; -DROP INDEX IF EXISTS idx_relprop_last_seen; -DROP TABLE relation_properties; +DROP INDEX IF EXISTS idx_edgetag_edge_id; +DROP INDEX IF EXISTS idx_edgetag_ttype; +DROP INDEX IF EXISTS idx_edgetag_last_seen; +DROP TABLE edge_tags; -DROP INDEX IF EXISTS idx_rel_to_entity_id; -DROP INDEX IF EXISTS idx_rel_from_entity_id; -DROP INDEX IF EXISTS idx_rel_rtype; -DROP INDEX IF EXISTS idx_rel_last_seen; -DROP TABLE relations; +DROP INDEX IF EXISTS idx_edge_to_entity_id; +DROP INDEX IF EXISTS idx_edge_from_entity_id; +DROP INDEX IF EXISTS idx_edge_etype; +DROP INDEX IF EXISTS idx_edge_last_seen; +DROP TABLE edges; -DROP INDEX IF EXISTS idx_entprop_entity_id; -DROP INDEX IF EXISTS idx_entprop_ptype; +DROP INDEX IF EXISTS idx_enttag_last_seen; +DROP INDEX IF EXISTS idx_enttag_ttype; DROP INDEX IF EXISTS idx_entprop_last_seen; -DROP TABLE entity_properties; +DROP TABLE entity_tags; DROP INDEX IF EXISTS idx_entities_etype; DROP INDEX IF EXISTS idx_entities_last_seen; diff --git a/repository/edge.go b/repository/edge.go new file mode 100644 index 0000000..091b01a --- /dev/null +++ b/repository/edge.go @@ -0,0 +1,204 @@ +// Copyright © by Jeff Foley 2017-2024. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. +// SPDX-License-Identifier: Apache-2.0 + +package repository + +import ( + "fmt" + "log" + "strconv" + "time" + + "github.com/owasp-amass/asset-db/types" + oam "github.com/owasp-amass/open-asset-model" +) + +// DeleteEdge removes an edge in the database by its ID. +// It takes a string representing the edge ID and removes the corresponding edge from the database. +// Returns an error if the edge is not found. +func (sql *sqlRepository) DeleteEdge(id string) error { + relId, err := strconv.ParseUint(id, 10, 64) + if err != nil { + return err + } + return sql.deleteEdges([]uint64{relId}) +} + +// deleteEdges removes all rows in the Edges table with primary keys in the provided slice. +func (sql *sqlRepository) deleteEdges(ids []uint64) error { + return sql.db.Exec("DELETE FROM edges WHERE edge_id IN ?", ids).Error +} + +// Link creates an edge between two entities in the database. +// It takes the source entity, the edge, and destination entity as inputs. +// The edge is established by creating a new Edge struct in the database, linking the two entities. +// Returns the created edge as a types.Edge or an error if the link creation fails. +func (sql *sqlRepository) Link(source *types.Entity, edge *types.Edge, destination *types.Entity) (*types.Edge, error) { + // check that this link will create a valid relationship within the taxonomy + srctype := source.Asset.AssetType() + destype := destination.Asset.AssetType() + if !oam.ValidRelationship(srctype, edge, destype) { + return &types.Edge{}, fmt.Errorf("%s -%s-> %s is not valid in the taxonomy", srctype, edge, destype) + } + + // ensure that duplicate relationships are not entered into the database + if rel, found := sql.isDuplicateEdge(source, edge, destination); found { + return rel, nil + } + + fromEntityId, err := strconv.ParseUint(source.ID, 10, 64) + if err != nil { + return &types.Edge{}, err + } + + toEntityId, err := strconv.ParseUint(destination.ID, 10, 64) + if err != nil { + return &types.Edge{}, err + } + + jsonContent, err := edge.Relation.JSON() + if err != nil { + return &types.Edge{}, err + } + + r := Edge{ + Type: string(edge.Relation.RelationType()), + Content: jsonContent, + FromEntityID: fromEntityId, + ToEntityID: toEntityId, + } + + result := sql.db.Create(&r) + if result.Error != nil { + return &types.Edge{}, result.Error + } + + return toEdge(r), nil +} + +// isDuplicateEdge checks if the relationship between source and dest already exists. +func (sql *sqlRepository) isDuplicateEdge(source *types.Entity, edge *types.Edge, dest *types.Entity) (*types.Edge, bool) { + var dup bool + var rel *types.Edge + + if outs, err := sql.OutgoingEdges(source, time.Time{}, edge.Relation.Label()); err == nil { + for _, out := range outs { + if dest.ID == out.ToEntity.ID { + _ = sql.edgeSeen(out) + rel, err = sql.edgeById(out.ID) + if err != nil { + log.Println("[ERROR] failed when re-retrieving relation", err) + return nil, false + } + dup = true + break + } + } + } + return rel, dup +} + +// edgeSeen updates the last seen timestamp for the specified edge. +func (sql *sqlRepository) edgeSeen(rel *types.Edge) error { + id, err := strconv.ParseInt(rel.ID, 10, 64) + if err != nil { + return fmt.Errorf("failed to update last seen for ID %s could not parse id; err: %w", rel.ID, err) + } + + result := sql.db.Exec("UPDATE edges SET last_seen = current_timestamp WHERE edge_id = ?", id) + if result.Error != nil { + return result.Error + } + + return nil +} + +// IncomingEdges finds all edges pointing to the entity of the specified relation types and last seen after the since parameter. +// If since.IsZero(), the parameter will be ignored. +// If no relationTypes are specified, all outgoing eges are returned. +func (sql *sqlRepository) IncomingEdges(entity *types.Entity, since time.Time, relationTypes ...string) ([]*types.Edge, error) { + entityId, err := strconv.ParseInt(entity.ID, 10, 64) + if err != nil { + return nil, err + } + + edges := []Edge{} + if len(relationTypes) > 0 { + res := sql.db.Where("to_entity_id = ? AND etype IN ?", entityId, relationTypes).Find(&edges) + if res.Error != nil { + return nil, res.Error + } + } else { + res := sql.db.Where("to_entity_id = ?", entityId).Find(&edges) + if res.Error != nil { + return nil, res.Error + } + } + + return toEdges(edges), nil +} + +// OutgoingEdges finds all edges from the entity of the specified relation types and last seen after the since parameter. +// If since.IsZero(), the parameter will be ignored. +// If no relationTypes are specified, all outgoing edges are returned. +func (sql *sqlRepository) OutgoingEdges(entity *types.Entity, since time.Time, relationTypes ...string) ([]*types.Edge, error) { + entityId, err := strconv.ParseInt(entity.ID, 10, 64) + if err != nil { + return nil, err + } + + edges := []Edge{} + if len(relationTypes) > 0 { + res := sql.db.Where("from_entity_id = ? AND etype IN ?", entityId, relationTypes).Find(&edges) + if res.Error != nil { + return nil, res.Error + } + } else { + res := sql.db.Where("from_entity_id = ?", entityId).Find(&edges) + if res.Error != nil { + return nil, res.Error + } + } + + return toEdges(edges), nil +} + +func (sql *sqlRepository) edgeById(id string) (*types.Edge, error) { + rel := Edge{} + + result := sql.db.Where("edge_id = ?", id).First(&rel) + if result.Error != nil { + return nil, result.Error + } + + return toEdge(rel), nil +} + +// toEdge converts a database Edge to a types.Edge. +func toEdge(r Edge) *types.Edge { + rel := &types.Edge{ + ID: strconv.FormatUint(r.ID, 10), + Type: r.Type, + LastSeen: r.LastSeen, + FromEntity: &types.Entity{ + ID: strconv.FormatUint(r.FromEntityID, 10), + // Not joining to Asset to get Content + }, + ToEntity: &types.Entity{ + ID: strconv.FormatUint(r.ToEntityID, 10), + // Not joining to Asset to get Content + }, + } + return rel +} + +// toEdges converts a slice of database Edges to a slice of types.Edge structs. +func toEdges(edges []Edge) []*types.Edge { + var res []*types.Edge + + for _, r := range edges { + res = append(res, toEdge(r)) + } + return res +} diff --git a/repository/relation_test.go b/repository/edge_test.go similarity index 100% rename from repository/relation_test.go rename to repository/edge_test.go diff --git a/repository/entity.go b/repository/entity.go index 9696a4d..a2f702c 100644 --- a/repository/entity.go +++ b/repository/entity.go @@ -103,7 +103,7 @@ func (sql *sqlRepository) CreateEntity(assetData oam.Asset) (*types.Entity, erro if id, err := strconv.ParseUint(e.ID, 10, 64); err == nil { entity.ID = id entity.CreatedAt = e.CreatedAt - entity.LastSeen = e.LastSeen + entity.LastSeen = time.Now() break } } @@ -185,16 +185,16 @@ func (sql *sqlRepository) FindEntityByContent(assetData oam.Asset, since time.Ti } var storedEntities []*types.Entity - for _, entity := range entities { - assetData, err := entity.Parse() + for _, e := range entities { + assetData, err := e.Parse() if err != nil { return []*types.Entity{}, err } storedEntities = append(storedEntities, &types.Entity{ - ID: strconv.FormatUint(entity.ID, 10), - CreatedAt: entity.CreatedAt, - LastSeen: entity.LastSeen, + ID: strconv.FormatUint(e.ID, 10), + CreatedAt: e.CreatedAt, + LastSeen: e.LastSeen, Asset: assetData, }) } diff --git a/repository/entity_test.go b/repository/entity_test.go index a4cce6a..99b1c6e 100644 --- a/repository/entity_test.go +++ b/repository/entity_test.go @@ -331,54 +331,54 @@ func TestRepository(t *testing.T) { } if relation == nil { - t.Fatalf("failed to link entities: relation is nil") + t.Fatalf("failed to link entities: edge is nil") } incoming, err := store.IncomingRelations(destinationEntity, start, tc.relation) if err != nil { - t.Fatalf("failed to query incoming relations: %s", err) + t.Fatalf("failed to query incoming edges: %s", err) } if incoming == nil { - t.Fatalf("failed to query incoming relations: incoming relations is nil %s", err) + t.Fatalf("failed to query incoming edges: incoming edge is nil %s", err) } if incoming[0].Type != tc.relation { - t.Fatalf("failed to query incoming relations: expected relation %s, got %s", tc.relation, incoming[0].Type) + t.Fatalf("failed to query incoming edges: expected relation %s, got %s", tc.relation, incoming[0].Type) } if incoming[0].FromEntity.ID != sourceEntity.ID { - t.Fatalf("failed to query incoming relations: expected source entity id %s, got %v", sourceEntity.ID, incoming[0].FromEntity.ID) + t.Fatalf("failed to query incoming edges: expected source entity id %s, got %v", sourceEntity.ID, incoming[0].FromEntity.ID) } if incoming[0].ToEntity.ID != destinationEntity.ID { - t.Fatalf("failed to query incoming relations: expected destination entity id %s, got %s", destinationEntity.ID, incoming[0].ToEntity.ID) + t.Fatalf("failed to query incoming edges: expected destination entity id %s, got %s", destinationEntity.ID, incoming[0].ToEntity.ID) } outgoing, err := store.OutgoingRelations(sourceEntity, start, tc.relation) if err != nil { - t.Fatalf("failed to query outgoing relations: %s", err) + t.Fatalf("failed to query outgoing edges: %s", err) } if outgoing == nil { - t.Fatalf("failed to query outgoing relations: outgoing relations is nil") + t.Fatalf("failed to query outgoing edges: outgoing edge is nil") } if outgoing[0].Type != tc.relation { - t.Fatalf("failed to query outgoing relations: expected relation %s, got %s", tc.relation, outgoing[0].Type) + t.Fatalf("failed to query outgoing edges: expected edge %s, got %s", tc.relation, outgoing[0].Type) } if outgoing[0].FromEntity.ID != sourceEntity.ID { - t.Fatalf("failed to query outgoing relations: expected source entity id %s, got %s", sourceEntity.ID, outgoing[0].FromEntity.ID) + t.Fatalf("failed to query outgoing edges: expected source entity id %s, got %s", sourceEntity.ID, outgoing[0].FromEntity.ID) } if outgoing[0].ToEntity.ID != destinationEntity.ID { - t.Fatalf("failed to query outgoing relations: expected destination entity id %s, got %s", destinationEntity.ID, outgoing[0].ToEntity.ID) + t.Fatalf("failed to query outgoing edges: expected destination entity id %s, got %s", destinationEntity.ID, outgoing[0].ToEntity.ID) } err = store.DeleteRelation(relation.ID) if err != nil { - t.Fatalf("failed to delete relation: %s", err) + t.Fatalf("failed to delete edges: %s", err) } err = store.DeleteEntity(destinationEntity.ID) diff --git a/repository/models.go b/repository/models.go index f382249..c11f3ac 100644 --- a/repository/models.go +++ b/repository/models.go @@ -19,8 +19,8 @@ import ( "github.com/owasp-amass/open-asset-model/org" "github.com/owasp-amass/open-asset-model/people" oamreg "github.com/owasp-amass/open-asset-model/registration" + "github.com/owasp-amass/open-asset-model/relation" "github.com/owasp-amass/open-asset-model/service" - "github.com/owasp-amass/open-asset-model/source" "github.com/owasp-amass/open-asset-model/url" "gorm.io/datatypes" ) @@ -34,12 +34,12 @@ type Entity struct { Content datatypes.JSON } -// Relation represents a relationship between two entities stored in the database. -type Relation struct { - ID uint64 `gorm:"primaryKey;column:relation_id"` +// Edge represents a relationship between two entities stored in the database. +type Edge struct { + ID uint64 `gorm:"primaryKey;column:edge_id"` CreatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();"` LastSeen time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();"` - Type string `gorm:"column:rtype"` + Type string `gorm:"column:etype"` Content datatypes.JSON FromEntityID uint64 `gorm:"column:from_entity_id"` ToEntityID uint64 `gorm:"column:to_entity_id"` @@ -59,11 +59,6 @@ func (e *Entity) Parse() (oam.Asset, error) { err = json.Unmarshal(e.Content, &fqdn) asset = &fqdn - case string(oam.NetworkEndpoint): - var ne domain.NetworkEndpoint - - err = json.Unmarshal(e.Content, &ne) - asset = &ne case string(oam.IPAddress): var ip network.IPAddress @@ -89,11 +84,6 @@ func (e *Entity) Parse() (oam.Asset, error) { err = json.Unmarshal(e.Content, &ipnetrec) asset = &ipnetrec - case string(oam.SocketAddress): - var sa network.SocketAddress - - err = json.Unmarshal(e.Content, &sa) - asset = &sa case string(oam.DomainRecord): var dr oamreg.DomainRecord @@ -144,11 +134,6 @@ func (e *Entity) Parse() (oam.Asset, error) { err = json.Unmarshal(e.Content, &url) asset = &url - case string(oam.Source): - var src source.Source - - err = json.Unmarshal(e.Content, &src) - asset = &src case string(oam.Service): var serv service.Service @@ -178,10 +163,6 @@ func (e *Entity) JSONQuery() (*datatypes.JSONQueryExpression, error) { switch v := asset.(type) { case *domain.FQDN: return jsonQuery.Equals(v.Name, "name"), nil - case *domain.NetworkEndpoint: - return jsonQuery.Equals(v.Address, "address"), nil - case *network.SocketAddress: - return jsonQuery.Equals(v.Address.String(), "address"), nil case *network.IPAddress: return jsonQuery.Equals(v.Address.String(), "address"), nil case *network.AutonomousSystem: @@ -212,8 +193,6 @@ func (e *Entity) JSONQuery() (*datatypes.JSONQueryExpression, error) { return jsonQuery.Equals(v.SerialNumber, "serial_number"), nil case *url.URL: return jsonQuery.Equals(v.Raw, "url"), nil - case *source.Source: - return jsonQuery.Equals(v.Name, "name"), nil case *service.Service: return jsonQuery.Equals(v.Identifier, "identifier"), nil case *oamfile.File: @@ -222,3 +201,42 @@ func (e *Entity) JSONQuery() (*datatypes.JSONQueryExpression, error) { return nil, fmt.Errorf("unknown asset type: %s", e.Type) } + +// Parse parses the content of the edge into the corresponding Open Asset Model (OAM) relation type. +// It returns the parsed relation and an error, if any. +func (e *Edge) Parse() (oam.Relation, error) { + var err error + var rel oam.Relation + + switch e.Type { + case string(oam.BasicDNSRelation): + var bdr relation.BasicDNSRelation + + err = json.Unmarshal(e.Content, &bdr) + rel = &bdr + case string(oam.PortRelation): + var pr relation.PortRelation + + err = json.Unmarshal(e.Content, &pr) + rel = &pr + case string(oam.PrefDNSRelation): + var pdr relation.PrefDNSRelation + + err = json.Unmarshal(e.Content, &pdr) + rel = &pdr + case string(oam.SimpleRelation): + var sr relation.SimpleRelation + + err = json.Unmarshal(e.Content, &sr) + rel = &sr + case string(oam.SRVDNSRelation): + var sdr relation.SRVDNSRelation + + err = json.Unmarshal(e.Content, &sdr) + rel = &sdr + default: + return nil, fmt.Errorf("unknown relation type: %s", e.Type) + } + + return rel, err +} diff --git a/repository/models_test.go b/repository/models_test.go index 7f0f9d6..a989fbf 100644 --- a/repository/models_test.go +++ b/repository/models_test.go @@ -20,7 +20,6 @@ import ( "github.com/owasp-amass/open-asset-model/people" oamreg "github.com/owasp-amass/open-asset-model/registration" "github.com/owasp-amass/open-asset-model/service" - "github.com/owasp-amass/open-asset-model/source" "github.com/owasp-amass/open-asset-model/url" "gorm.io/datatypes" "gorm.io/gorm" @@ -40,10 +39,6 @@ func TestModels(t *testing.T) { description: "parse fqdn", asset: &domain.FQDN{Name: "www.example.com"}, }, - { - description: "parse network endpoint", - asset: &domain.NetworkEndpoint{Address: "www.example.com:80"}, - }, { description: "parse ip address", asset: &network.IPAddress{Address: ip, Type: "IPv4"}, @@ -76,10 +71,6 @@ func TestModels(t *testing.T) { description: "parse tls certificate", asset: &oamcert.TLSCertificate{SerialNumber: "25:89:5f:3b:96:c8:18:89:09:04:8b:6c:64:88:6f:1b"}, }, - { - description: "parse socket address", - asset: &network.SocketAddress{Address: netip.MustParseAddrPort("192.168.1.1:443")}, - }, { description: "parse person", asset: &people.Person{FullName: "John Doe"}, @@ -108,10 +99,6 @@ func TestModels(t *testing.T) { description: "parse contact record", asset: &contact.ContactRecord{DiscoveredAt: "https://owasp.org"}, }, - { - description: "parse source", - asset: &source.Source{Name: "https://www.owasp.org"}, - }, { description: "parse service", asset: &service.Service{Identifier: "12345"}, @@ -151,11 +138,6 @@ func TestModels(t *testing.T) { asset: &domain.FQDN{Name: "www.example.com"}, expectedQuery: datatypes.JSONQuery("content").Equals("www.example.com", "name"), }, - { - description: "json query for network endpoint", - asset: &domain.NetworkEndpoint{Address: "www.example.com:80"}, - expectedQuery: datatypes.JSONQuery("content").Equals("www.example.com:80", "address"), - }, { description: "json query for ip address", asset: &network.IPAddress{Address: ip, Type: "IPv4"}, @@ -181,11 +163,6 @@ func TestModels(t *testing.T) { asset: &network.AutonomousSystem{Number: 64496}, expectedQuery: datatypes.JSONQuery("content").Equals(64496, "number"), }, - { - description: "json query for socket address", - asset: &network.SocketAddress{Address: netip.MustParseAddrPort("192.168.1.1:443")}, - expectedQuery: datatypes.JSONQuery("content").Equals("192.168.1.1:443", "address"), - }, { description: "json query for person", asset: &people.Person{FullName: "John Doe"}, @@ -236,11 +213,6 @@ func TestModels(t *testing.T) { asset: &contact.ContactRecord{DiscoveredAt: "https://owasp.org"}, expectedQuery: datatypes.JSONQuery("content").Equals("https://owasp.org", "discovered_at"), }, - { - description: "json query for source", - asset: &source.Source{Name: "https://www.owasp.org"}, - expectedQuery: datatypes.JSONQuery("content").Equals("https://www.owasp.org", "name"), - }, { description: "json query for service", asset: &service.Service{Identifier: "12345"}, diff --git a/repository/relation.go b/repository/relation.go deleted file mode 100644 index 378aa5a..0000000 --- a/repository/relation.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright © by Jeff Foley 2017-2024. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. -// SPDX-License-Identifier: Apache-2.0 - -package repository - -import ( - "fmt" - "log" - "strconv" - "time" - - "github.com/owasp-amass/asset-db/types" - oam "github.com/owasp-amass/open-asset-model" -) - -// DeleteRelation removes a relation in the database by its ID. -// It takes a string representing the relation ID and removes the corresponding relation from the database. -// Returns an error if the relation is not found. -func (sql *sqlRepository) DeleteRelation(id string) error { - relId, err := strconv.ParseUint(id, 10, 64) - if err != nil { - return err - } - return sql.deleteRelations([]uint64{relId}) -} - -// deleteRelations removes all rows in the Relations table with primary keys in the provided slice. -func (sql *sqlRepository) deleteRelations(ids []uint64) error { - return sql.db.Exec("DELETE FROM relations WHERE relation_id IN ?", ids).Error -} - -// Link creates a relation between two entities in the database. -// It takes the source entity, relation type, and destination entity as inputs. -// The relation is established by creating a new Relation struct in the database, linking the two entities. -// Returns the created relation as a types.Relation or an error if the link creation fails. -func (sql *sqlRepository) Link(source *types.Entity, relation string, destination *types.Entity) (*types.Relation, error) { - // check that this link will create a valid relationship within the taxonomy - srctype := source.Asset.AssetType() - destype := destination.Asset.AssetType() - if !oam.ValidRelationship(srctype, relation, destype) { - return &types.Relation{}, fmt.Errorf("%s -%s-> %s is not valid in the taxonomy", srctype, relation, destype) - } - - // ensure that duplicate relationships are not entered into the database - if rel, found := sql.isDuplicateRelation(source, relation, destination); found { - return rel, nil - } - - fromEntityId, err := strconv.ParseUint(source.ID, 10, 64) - if err != nil { - return &types.Relation{}, err - } - - toEntityId, err := strconv.ParseUint(destination.ID, 10, 64) - if err != nil { - return &types.Relation{}, err - } - - r := Relation{ - Type: relation, - FromEntityID: fromEntityId, - ToEntityID: toEntityId, - } - - result := sql.db.Create(&r) - if result.Error != nil { - return &types.Relation{}, result.Error - } - - return toRelation(r), nil -} - -// isDuplicateRelation checks if the relationship between source and dest already exists. -func (sql *sqlRepository) isDuplicateRelation(source *types.Entity, relation string, dest *types.Entity) (*types.Relation, bool) { - var dup bool - var rel *types.Relation - - if outs, err := sql.OutgoingRelations(source, time.Time{}, relation); err == nil { - for _, out := range outs { - if dest.ID == out.ToEntity.ID { - _ = sql.relationSeen(out) - rel, err = sql.relationById(out.ID) - if err != nil { - log.Println("[ERROR] failed when re-retrieving relation", err) - return nil, false - } - dup = true - break - } - } - } - return rel, dup -} - -// updateRelationLastSeen updates the last seen timestamp for the specified relation. -func (sql *sqlRepository) relationSeen(rel *types.Relation) error { - id, err := strconv.ParseInt(rel.ID, 10, 64) - if err != nil { - return fmt.Errorf("failed to update last seen for ID %s could not parse id; err: %w", rel.ID, err) - } - - result := sql.db.Exec("UPDATE relations SET last_seen = current_timestamp WHERE relation_id = ?", id) - if result.Error != nil { - return result.Error - } - - return nil -} - -// IncomingRelations finds all relations pointing to the entity of the specified relation types and last seen after the since parameter. -// If since.IsZero(), the parameter will be ignored. -// If no relationTypes are specified, all outgoing relations are returned. -func (sql *sqlRepository) IncomingRelations(entity *types.Entity, since time.Time, relationTypes ...string) ([]*types.Relation, error) { - entityId, err := strconv.ParseInt(entity.ID, 10, 64) - if err != nil { - return nil, err - } - - relations := []Relation{} - if len(relationTypes) > 0 { - res := sql.db.Where("to_entity_id = ? AND rtype IN ?", entityId, relationTypes).Find(&relations) - if res.Error != nil { - return nil, res.Error - } - } else { - res := sql.db.Where("to_entity_id = ?", entityId).Find(&relations) - if res.Error != nil { - return nil, res.Error - } - } - - return toRelations(relations), nil -} - -// OutgoingRelations finds all relations from the entity of the specified relation types and last seen after the since parameter. -// If since.IsZero(), the parameter will be ignored. -// If no relationTypes are specified, all outgoing relations are returned. -func (sql *sqlRepository) OutgoingRelations(entity *types.Entity, since time.Time, relationTypes ...string) ([]*types.Relation, error) { - entityId, err := strconv.ParseInt(entity.ID, 10, 64) - if err != nil { - return nil, err - } - - relations := []Relation{} - if len(relationTypes) > 0 { - res := sql.db.Where("from_entity_id = ? AND rtype IN ?", entityId, relationTypes).Find(&relations) - if res.Error != nil { - return nil, res.Error - } - } else { - res := sql.db.Where("from_entity_id = ?", entityId).Find(&relations) - if res.Error != nil { - return nil, res.Error - } - } - - return toRelations(relations), nil -} - -func (sql *sqlRepository) relationById(id string) (*types.Relation, error) { - rel := Relation{} - - result := sql.db.Where("relation_id = ?", id).First(&rel) - if result.Error != nil { - return nil, result.Error - } - - return toRelation(rel), nil -} - -// toRelation converts a database Relation to a types.Relation. -func toRelation(r Relation) *types.Relation { - rel := &types.Relation{ - ID: strconv.FormatUint(r.ID, 10), - Type: r.Type, - LastSeen: r.LastSeen, - FromEntity: &types.Entity{ - ID: strconv.FormatUint(r.FromEntityID, 10), - // Not joining to Asset to get Content - }, - ToEntity: &types.Entity{ - ID: strconv.FormatUint(r.ToEntityID, 10), - // Not joining to Asset to get Content - }, - } - return rel -} - -// toRelations converts a slice database Relations to a slice of types.Relation structs. -func toRelations(relations []Relation) []*types.Relation { - var res []*types.Relation - - for _, r := range relations { - res = append(res, toRelation(r)) - } - - return res -} diff --git a/repository/repository.go b/repository/repository.go index 2a6f39c..06de96e 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -18,13 +18,13 @@ type Repository interface { CreateEntity(asset oam.Asset) (*types.Entity, error) UpdateEntityLastSeen(id string) error DeleteEntity(id string) error - DeleteRelation(id string) error + DeleteEdge(id string) error FindEntityById(id string, since time.Time) (*types.Entity, error) FindEntityByContent(asset oam.Asset, since time.Time) ([]*types.Entity, error) FindEntitiesByType(atype oam.AssetType, since time.Time) ([]*types.Entity, error) FindEntitiesByScope(constraints []oam.Asset, since time.Time) ([]*types.Entity, error) - Link(source *types.Entity, relation string, destination *types.Entity) (*types.Relation, error) - IncomingRelations(asset *types.Entity, since time.Time, relationTypes ...string) ([]*types.Relation, error) - OutgoingRelations(asset *types.Entity, since time.Time, relationTypes ...string) ([]*types.Relation, error) + Link(source *types.Entity, edge *types.Edge, destination *types.Entity) (*types.Edge, error) + IncomingEdges(asset *types.Entity, since time.Time, relationTypes ...string) ([]*types.Edge, error) + OutgoingEdges(asset *types.Entity, since time.Time, relationTypes ...string) ([]*types.Edge, error) Close() error } diff --git a/repository/sql_scope.go b/repository/sql_scope.go index b6d13c5..adbd376 100644 --- a/repository/sql_scope.go +++ b/repository/sql_scope.go @@ -56,13 +56,13 @@ func (sql *sqlRepository) inAndOut(constraint oam.Asset, since time.Time) ([]*ty ids := stringset.New() for _, constraint := range constraints { - if rels, err := sql.IncomingRelations(constraint, since); err == nil { + if rels, err := sql.IncomingEdges(constraint, since); err == nil { for _, rel := range rels { ids.Insert(rel.FromEntity.ID) } } - if rels, err := sql.OutgoingRelations(constraint, since); err == nil { + if rels, err := sql.OutgoingEdges(constraint, since); err == nil { for _, rel := range rels { ids.Insert(rel.ToEntity.ID) } diff --git a/types/types.go b/types/types.go index b9a5424..1c4c666 100644 --- a/types/types.go +++ b/types/types.go @@ -11,7 +11,6 @@ import ( ) // Entity represents an entity in the asset database. -// It contains an ID and the corresponding oam.Asset. type Entity struct { ID string CreatedAt time.Time @@ -19,13 +18,26 @@ type Entity struct { Asset oam.Asset } -// Relation represents a relationship between two entities in the asset database. -// It contains an ID, a type describing the relationship, and references to the source and destination entities. -type Relation struct { +// EntityTag represents addition metadata added to an entity in the asset database. +type EntityTag struct { + ID string + CreatedAt time.Time + LastSeen time.Time +} + +// Edge represents a relationship between two entities in the asset database. +type Edge struct { ID string - Type string CreatedAt time.Time LastSeen time.Time + Relation oam.Relation FromEntity *Entity ToEntity *Entity } + +// EdgeTag represents addition metadata added to an entity in the asset database. +type EdgeTag struct { + ID string + CreatedAt time.Time + LastSeen time.Time +}