From b44f441c825b8534b894857c983f0df7f1508546 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 24 Oct 2022 11:12:12 -0400 Subject: [PATCH] Upgrade generic cataloger (#1281) * add second generation of generic cataloger Signed-off-by: Alex Goodman * upgrade aplm cataloger to use generic.Cataloger Signed-off-by: Alex Goodman * remove pacakge found-by attribute from the definition of a package ID Signed-off-by: Alex Goodman Signed-off-by: Alex Goodman --- .../TestCycloneDxDirectoryEncoder.golden | 8 +- .../snapshot/TestCycloneDxImageEncoder.golden | 14 +- .../stereoscope-fixture-image-simple.golden | Bin 15360 -> 15360 bytes .../TestCycloneDxDirectoryEncoder.golden | 8 +- .../snapshot/TestCycloneDxImageEncoder.golden | 14 +- .../stereoscope-fixture-image-simple.golden | Bin 15360 -> 15360 bytes .../TestSPDXJSONDirectoryEncoder.golden | 10 +- .../snapshot/TestSPDXJSONImageEncoder.golden | 10 +- .../snapshot/TestSPDXRelationshipOrder.golden | 20 +-- .../stereoscope-fixture-image-simple.golden | Bin 15360 -> 15360 bytes .../snapshot/TestSPDXJSONSPDXIDs.golden | 12 +- .../TestSPDXTagValueDirectoryEncoder.golden | 10 +- .../TestSPDXTagValueImageEncoder.golden | 10 +- .../stereoscope-fixture-image-simple.golden | Bin 15360 -> 15360 bytes .../snapshot/TestDirectoryEncoder.golden | 4 +- .../TestEncodeFullJSONDocument.golden | 4 +- .../snapshot/TestImageEncoder.golden | 20 +-- .../stereoscope-fixture-image-simple.golden | Bin 15360 -> 15360 bytes syft/linux/identify_release.go | 12 +- syft/pkg/alpm_metadata.go | 32 +--- syft/pkg/apk_metadata.go | 2 +- syft/pkg/cataloger/alpm/cataloger.go | 43 +---- syft/pkg/cataloger/alpm/package.go | 52 ++++++ .../alpm/package_test.go} | 37 +++-- syft/pkg/cataloger/alpm/parse_alpm_db.go | 106 ++++++------ syft/pkg/cataloger/generic/cataloger.go | 151 ++++++++++++++++++ syft/pkg/cataloger/generic/cataloger_test.go | 87 ++++++++++ syft/pkg/cataloger/generic/parser.go | 14 ++ .../generic/test-fixtures/a-path.txt | 1 + .../generic/test-fixtures/another-path.txt | 1 + .../cataloger/generic/test-fixtures/empty.txt | 0 .../generic/test-fixtures/last/path.txt | 1 + syft/pkg/dpkg_metadata.go | 2 +- syft/pkg/package.go | 2 +- syft/pkg/package_test.go | 8 - syft/pkg/rpm_metadata.go | 2 +- syft/pkg/url.go | 2 +- syft/pkg/url_test.go | 19 +-- syft/source/location.go | 7 + syft/source/location_read_closer.go | 15 ++ 40 files changed, 495 insertions(+), 245 deletions(-) create mode 100644 syft/pkg/cataloger/alpm/package.go rename syft/pkg/{alpm_metadata_test.go => cataloger/alpm/package_test.go} (80%) create mode 100644 syft/pkg/cataloger/generic/cataloger.go create mode 100644 syft/pkg/cataloger/generic/cataloger_test.go create mode 100644 syft/pkg/cataloger/generic/parser.go create mode 100644 syft/pkg/cataloger/generic/test-fixtures/a-path.txt create mode 100644 syft/pkg/cataloger/generic/test-fixtures/another-path.txt create mode 100644 syft/pkg/cataloger/generic/test-fixtures/empty.txt create mode 100644 syft/pkg/cataloger/generic/test-fixtures/last/path.txt create mode 100644 syft/source/location_read_closer.go diff --git a/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden b/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden index 3b23a84e516..35596741003 100644 --- a/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden +++ b/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden @@ -1,10 +1,10 @@ { "bomFormat": "CycloneDX", "specVersion": "1.4", - "serialNumber": "urn:uuid:3ea3363f-3945-4859-9ba1-9a395983d248", + "serialNumber": "urn:uuid:f426926b-4867-4b52-9142-23997f685f2c", "version": 1, "metadata": { - "timestamp": "2022-05-23T12:05:00-07:00", + "timestamp": "2022-10-24T09:54:37-04:00", "tools": [ { "vendor": "anchore", @@ -20,7 +20,7 @@ }, "components": [ { - "bom-ref": "b85dbb4e6ece5082", + "bom-ref": "e624319940d8d36a", "type": "library", "name": "package-1", "version": "1.0.1", @@ -57,7 +57,7 @@ ] }, { - "bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=ceda99598967ae8d", + "bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=b8645f4ac2a0891e", "type": "library", "name": "package-2", "version": "2.0.1", diff --git a/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden b/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden index 6dac17e18d5..984423df093 100644 --- a/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden +++ b/syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden @@ -1,10 +1,10 @@ { "bomFormat": "CycloneDX", "specVersion": "1.4", - "serialNumber": "urn:uuid:c825402b-bbfa-4ad5-81b1-6a8332a6a8b6", + "serialNumber": "urn:uuid:41bbbcc7-694d-4b07-a678-0afb67dabdf9", "version": 1, "metadata": { - "timestamp": "2022-05-23T12:05:01-07:00", + "timestamp": "2022-10-24T09:54:37-04:00", "tools": [ { "vendor": "anchore", @@ -13,7 +13,7 @@ } ], "component": { - "bom-ref": "e779c1ed804ba529", + "bom-ref": "522dc6b135a55bb4", "type": "container", "name": "user-image-input", "version": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" @@ -21,7 +21,7 @@ }, "components": [ { - "bom-ref": "2a46171f91c8d4bc", + "bom-ref": "5ffee24fb164cffc", "type": "library", "name": "package-1", "version": "1.0.1", @@ -53,7 +53,7 @@ }, { "name": "syft:location:0:layerID", - "value": "sha256:cd8f3884f1211d65c19ce5bbc5174bcd2ce8ba96b63e5b3693969a53279c4405" + "value": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59" }, { "name": "syft:location:0:path", @@ -62,7 +62,7 @@ ] }, { - "bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=ae77680e9b1d087e", + "bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=8b16570b2b4155c3", "type": "library", "name": "package-2", "version": "2.0.1", @@ -83,7 +83,7 @@ }, { "name": "syft:location:0:layerID", - "value": "sha256:42d2ea51c688e6dc7be81a305acbe006d27a6ef0c26ae3888fd0d4ce44f69265" + "value": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec" }, { "name": "syft:location:0:path", diff --git a/syft/formats/cyclonedxjson/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden b/syft/formats/cyclonedxjson/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden index 3d93b6d3ad1c86098e440073bcf8cfdf5eea53e1..11a1958c8935029370bb6a48ad18899403776949 100644 GIT binary patch literal 15360 zcmeHOZExE)5ccQ&3Qzl**nFpCU>~}sKnoO0(Pka6qA2)sOtjjPA<0F9Apdm%%?PJlu^wTR3f}9JyT$INc2+m0LY2Bo z+kCjL@($GhDlwJ2$Qy9Ytb$~en2A>+IZ9kvg#0`iT_ul-cgg5g^6bIcA**Z}Z~(B(HF&Ar5m|{4vYNJ{c9W@%S2tt-v&Z z<%AK&j!Ak<=oup;l8qP}NRkOgHD&)`*NePv%IXqzyV}xCq&m%{q*>c-Il>!@J(C9~ zC;gC*;g}AZ_f3CTmlIei|JJvyc5m}$Zjs3ZiUxaqxH<5pdka}WC)5Q(m;?gGWhr0tP6t%jdrI6?%$4P|rP|4?pM&h_=5YAJNDFF!+e z=O#b-`rq0AFCRUB@#OT+yZo?a|3~-pb^kM>h>ZLHT{hiS9UXLze0HTul=BrV(0Tsv zApVzZWBq4Da24nOAzp{8ByWT5uCD>cd{a{`qhswgO%cpt?3XzyKzi#9rAlcoiC`%g zUKpNQNsP1tFy!MF(O;Tr$E;^4>mDcoaY|$I0;)H+h~;rvH-(wBfi4O=DJR(Aw)|HO zE&9Nk2Tfz##f!!xI2i>qu3_a~Im1wY^Cnc-b$RowK?Je9KAl3bE@xgkEgUMbY%|AE zEzCJe9!{2s=O}qHG3QG~UzX$91TYj|G+M(-sO_VosV=8w-a?Q?i#70clsq*S#>?(4 zl)6E1->^M)ge&iF&=!2&*P@?*$}3RoyMrrTC;o7bBnK^r^2y(p_)mNk|Fby_6#1V| zT6A|6H-yKWXV{(GRY_a?k3>Mb{}ZMtG5}Qf2&I}M{yRuy`jg|oIoZYf?JkPDcm3 z?))!NY%~7@`QP~c=OFLQlK<`c8X%c=%h}RQ6HMD^GdBokZnS^%#EZz(ejsi)nA)#M zAa-^9-uOQf`JY2905|;KJOOGyBW~ya)JOaurVB>?Z$Iu>liFYw-8=)Xe_`qy8gQ*i zEJO=L3q%V<3;ci&PqIn?_abld!Ax4Gf-TyUKK3XcKB_mIfGV=884#+q6MM_z9S3#2P@x!egFUf literal 15360 zcmeHOTW{Mo6wdR0g{Qs7Ht(bi?4fH4v_P>GZPo!Rih$xJ(P~SEBo_&S{P!Knj$?am z)kLz<3k3t>^@#iqk90mB8hfg2qL?y9Ag3+2)O%%-k1=6b3(TYz+QdS}i7?nu&ZWUT z<^=Ir$#{I!ej&t=cx7{l9(o`pt{eKfg3wgb@-9^t!xu?@&s&^-mBifTKZYoqTCfcXdCz{znHv`3~)H z|HcsLZ2$k3dDb$g7W>B>i(UH{6bJinfwhBff zwKA4_DW$hqdvGlzPZCF&N<{2pNC@I;XT!z@dL`Ox?m?C*oUU9wFoFmkVmw6j46zBr z6Q)L-63h|*r*4>Ml}f0eFS);y^yZ7$zb zHqD}m*VED+N9(eD0aMv6bOm%>6`obs>-6c|W!rL^+-YWD5UcjjvX9XuTTG|N(bE~s zY?*v9(-*6OzRjnLnJe#V{?3aJX?Agvu15PTtBUJ+p4JdA`<`ZhEYj&()JvVX>8kr} zgUq*&$Ghm&le6c)j~8Vzp5{hR$7Pacla2g3dtcEMEwhk}-aiJ=F|3S-BB~2>nO4qJ zi=tMUo>{>_Ob(bw(==OLMYp|}~ZU%O&d`_P}gTHgN*9KfCTKjs{T_y1=B z?XjZFXU?ZnHzcF#s_IejhNu?(BSaegQ;9GWk^uk1k^%oah-dnf`~OSb59;qIe7(zG zH`ErA0fB%(Kp-IS10x`Tomqxs=_#?)VM>uVf|g`oH}@hGqEvA7a_vM4zEG_k6lxu01ieEmm8Tuq_f>)7GzL{+DT4Awd*?=uU+9~{v|i>P=K#&uiM+@U;@A zWK-|YQww$PqViG`E+)n*2nQ(UF~QghZg6ZIY}gnM4tZiMF-|2q7KxxxOhrs%5o_3v zk+ES6age%_#E2zMb8LiCPFN!or?94oYm+#L23P`NCFhYLLOXb*@)lWU9AjR@MDXbS z?RQ9~4`Hx9Kcnkr-Cp8#YC1#Q$J_H}L;YXsiDs9O8fdgZdpw z>}>zt{eL6~-r4^p2nGNDAggE<|LeL8Y8QF!xdWR-Lj*B9e1_1Z`Gr{Bmtu7r9{Zua z-mZ$@PyB~SN&^2s#0y}X^zP@0?eU-H{Kr=OhX?)dM;SX(E7(VAc0mT0x?G2Wb{d6j zKp-Fx5C{nTa0twFmU>rKUlYf9?Bv_N|D4miGvW0K@#g#o4&VRB!u_Cr5IuRiIyo8j z5(@+-_}b0$Gkvj)EL5&=d1>;w8~XIBK07x|>qC#jGM&w*ZlWNvC?LRqlu9gw1(=sw z!P(Fl!UQZvUO+92oh06y82)^Mq_M(b<&h)A@T5Nf_p&~N);z91Is^0Q?`#}|Gv5F5 zAiW%_aRIt2Q;4LcuH4r%+w7m*|2OE){)Gtkzb|!c$-Rln77MLohn - + - 2022-05-23T12:02:41-07:00 + 2022-10-24T09:54:54-04:00 anchore @@ -14,7 +14,7 @@ - + package-1 1.0.1 @@ -32,7 +32,7 @@ /some/path/pkg1 - + package-2 2.0.1 cpe:2.3:*:some:package:2:*:*:*:*:*:*:* diff --git a/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden b/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden index 6ef8367e66a..1783aba0dcb 100644 --- a/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden +++ b/syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden @@ -1,7 +1,7 @@ - + - 2022-05-23T12:02:42-07:00 + 2022-10-24T09:54:54-04:00 anchore @@ -9,13 +9,13 @@ v0.42.0-bogus - + user-image-input sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368 - + package-1 1.0.1 @@ -30,11 +30,11 @@ python PythonPackageMetadata python - sha256:cd8f3884f1211d65c19ce5bbc5174bcd2ce8ba96b63e5b3693969a53279c4405 + sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59 /somefile-1.txt - + package-2 2.0.1 cpe:2.3:*:some:package:2:*:*:*:*:*:*:* @@ -43,7 +43,7 @@ the-cataloger-2 DpkgMetadata deb - sha256:42d2ea51c688e6dc7be81a305acbe006d27a6ef0c26ae3888fd0d4ce44f69265 + sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec /somefile-2.txt 0 diff --git a/syft/formats/cyclonedxxml/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden b/syft/formats/cyclonedxxml/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden index 3d93b6d3ad1c86098e440073bcf8cfdf5eea53e1..11a1958c8935029370bb6a48ad18899403776949 100644 GIT binary patch literal 15360 zcmeHOZExE)5ccQ&3Qzl**nFpCU>~}sKnoO0(Pka6qA2)sOtjjPA<0F9Apdm%%?PJlu^wTR3f}9JyT$INc2+m0LY2Bo z+kCjL@($GhDlwJ2$Qy9Ytb$~en2A>+IZ9kvg#0`iT_ul-cgg5g^6bIcA**Z}Z~(B(HF&Ar5m|{4vYNJ{c9W@%S2tt-v&Z z<%AK&j!Ak<=oup;l8qP}NRkOgHD&)`*NePv%IXqzyV}xCq&m%{q*>c-Il>!@J(C9~ zC;gC*;g}AZ_f3CTmlIei|JJvyc5m}$Zjs3ZiUxaqxH<5pdka}WC)5Q(m;?gGWhr0tP6t%jdrI6?%$4P|rP|4?pM&h_=5YAJNDFF!+e z=O#b-`rq0AFCRUB@#OT+yZo?a|3~-pb^kM>h>ZLHT{hiS9UXLze0HTul=BrV(0Tsv zApVzZWBq4Da24nOAzp{8ByWT5uCD>cd{a{`qhswgO%cpt?3XzyKzi#9rAlcoiC`%g zUKpNQNsP1tFy!MF(O;Tr$E;^4>mDcoaY|$I0;)H+h~;rvH-(wBfi4O=DJR(Aw)|HO zE&9Nk2Tfz##f!!xI2i>qu3_a~Im1wY^Cnc-b$RowK?Je9KAl3bE@xgkEgUMbY%|AE zEzCJe9!{2s=O}qHG3QG~UzX$91TYj|G+M(-sO_VosV=8w-a?Q?i#70clsq*S#>?(4 zl)6E1->^M)ge&iF&=!2&*P@?*$}3RoyMrrTC;o7bBnK^r^2y(p_)mNk|Fby_6#1V| zT6A|6H-yKWXV{(GRY_a?k3>Mb{}ZMtG5}Qf2&I}M{yRuy`jg|oIoZYf?JkPDcm3 z?))!NY%~7@`QP~c=OFLQlK<`c8X%c=%h}RQ6HMD^GdBokZnS^%#EZz(ejsi)nA)#M zAa-^9-uOQf`JY2905|;KJOOGyBW~ya)JOaurVB>?Z$Iu>liFYw-8=)Xe_`qy8gQ*i zEJO=L3q%V<3;ci&PqIn?_abld!Ax4Gf-TyUKK3XcKB_mIfGV=884#+q6MM_z9S3#2P@x!egFUf literal 15360 zcmeHOTW{Mo6wdR0g{Qs7Ht(bi?4fH4v_P>GZPo!Rih$xJ(P~SEBo_&S{P!Knj$?am z)kLz<3k3t>^@#iqk90mB8hfg2qL?y9Ag3+2)O%%-k1=6b3(TYz+QdS}i7?nu&ZWUT z<^=Ir$#{I!ej&t=cx7{l9(o`pt{eKfg3wgb@-9^t!xu?@&s&^-mBifTKZYoqTCfcXdCz{znHv`3~)H z|HcsLZ2$k3dDb$g7W>B>i(UH{6bJinfwhBff zwKA4_DW$hqdvGlzPZCF&N<{2pNC@I;XT!z@dL`Ox?m?C*oUU9wFoFmkVmw6j46zBr z6Q)L-63h|*r*4>Ml}f0eFS);y^yZ7$zb zHqD}m*VED+N9(eD0aMv6bOm%>6`obs>-6c|W!rL^+-YWD5UcjjvX9XuTTG|N(bE~s zY?*v9(-*6OzRjnLnJe#V{?3aJX?Agvu15PTtBUJ+p4JdA`<`ZhEYj&()JvVX>8kr} zgUq*&$Ghm&le6c)j~8Vzp5{hR$7Pacla2g3dtcEMEwhk}-aiJ=F|3S-BB~2>nO4qJ zi=tMUo>{>_Ob(bw(==OLMYp|}~ZU%O&d`_P}gTHgN*9KfCTKjs{T_y1=B z?XjZFXU?ZnHzcF#s_IejhNu?(BSaegQ;9GWk^uk1k^%oah-dnf`~OSb59;qIe7(zG zH`ErA0fB%(Kp-IS10x`Tomqxs=_#?)VM>uVf|g`oH}@hGqEvA7a_vM4zEG_k6lxu01ieEmm8Tuq_f>)7GzL{+DT4Awd*?=uU+9~{v|i>P=K#&uiM+@U;@A zWK-|YQww$PqViG`E+)n*2nQ(UF~QghZg6ZIY}gnM4tZiMF-|2q7KxxxOhrs%5o_3v zk+ES6age%_#E2zMb8LiCPFN!or?94oYm+#L23P`NCFhYLLOXb*@)lWU9AjR@MDXbS z?RQ9~4`Hx9Kcnkr-Cp8#YC1#Q$J_H}L;YXsiDs9O8fdgZdpw z>}>zt{eL6~-r4^p2nGNDAggE<|LeL8Y8QF!xdWR-Lj*B9e1_1Z`Gr{Bmtu7r9{Zua z-mZ$@PyB~SN&^2s#0y}X^zP@0?eU-H{Kr=OhX?)dM;SX(E7(VAc0mT0x?G2Wb{d6j zKp-Fx5C{nTa0twFmU>rKUlYf9?Bv_N|D4miGvW0K@#g#o4&VRB!u_Cr5IuRiIyo8j z5(@+-_}b0$Gkvj)EL5&=d1>;w8~XIBK07x|>qC#jGM&w*ZlWNvC?LRqlu9gw1(=sw z!P(Fl!UQZvUO+92oh06y82)^Mq_M(b<&h)A@T5Nf_p&~N);z91Is^0Q?`#}|Gv5F5 zAiW%_aRIt2Q;4LcuH4r%+w7m*|2OE){)Gtkzb|!c$-Rln77MLohn~}sKnoO0(Pka6qA2)sOtjjPA<0F9Apdm%%?PJlu^wTR3f}9JyT$INc2+m0LY2Bo z+kCjL@($GhDlwJ2$Qy9Ytb$~en2A>+IZ9kvg#0`iT_ul-cgg5g^6bIcA**Z}Z~(B(HF&Ar5m|{4vYNJ{c9W@%S2tt-v&Z z<%AK&j!Ak<=oup;l8qP}NRkOgHD&)`*NePv%IXqzyV}xCq&m%{q*>c-Il>!@J(C9~ zC;gC*;g}AZ_f3CTmlIei|JJvyc5m}$Zjs3ZiUxaqxH<5pdka}WC)5Q(m;?gGWhr0tP6t%jdrI6?%$4P|rP|4?pM&h_=5YAJNDFF!+e z=O#b-`rq0AFCRUB@#OT+yZo?a|3~-pb^kM>h>ZLHT{hiS9UXLze0HTul=BrV(0Tsv zApVzZWBq4Da24nOAzp{8ByWT5uCD>cd{a{`qhswgO%cpt?3XzyKzi#9rAlcoiC`%g zUKpNQNsP1tFy!MF(O;Tr$E;^4>mDcoaY|$I0;)H+h~;rvH-(wBfi4O=DJR(Aw)|HO zE&9Nk2Tfz##f!!xI2i>qu3_a~Im1wY^Cnc-b$RowK?Je9KAl3bE@xgkEgUMbY%|AE zEzCJe9!{2s=O}qHG3QG~UzX$91TYj|G+M(-sO_VosV=8w-a?Q?i#70clsq*S#>?(4 zl)6E1->^M)ge&iF&=!2&*P@?*$}3RoyMrrTC;o7bBnK^r^2y(p_)mNk|Fby_6#1V| zT6A|6H-yKWXV{(GRY_a?k3>Mb{}ZMtG5}Qf2&I}M{yRuy`jg|oIoZYf?JkPDcm3 z?))!NY%~7@`QP~c=OFLQlK<`c8X%c=%h}RQ6HMD^GdBokZnS^%#EZz(ejsi)nA)#M zAa-^9-uOQf`JY2905|;KJOOGyBW~ya)JOaurVB>?Z$Iu>liFYw-8=)Xe_`qy8gQ*i zEJO=L3q%V<3;ci&PqIn?_abld!Ax4Gf-TyUKK3XcKB_mIfGV=884#+q6MM_z9S3#2P@x!egFUf literal 15360 zcmeHOTW{Mo6wdR0g{Qs7Ht(bi?4fH4v_P>GZPo!Rih$xJ(P~SEBo_&S{P!Knj$?am z)kLz<3k3t>^@#iqk90mB8hfg2qL?y9Ag3+2)O%%-k1=6b3(TYz+QdS}i7?nu&ZWUT z<^=Ir$#{I!ej&t=cx7{l9(o`pt{eKfg3wgb@-9^t!xu?@&s&^-mBifTKZYoqTCfcXdCz{znHv`3~)H z|HcsLZ2$k3dDb$g7W>B>i(UH{6bJinfwhBff zwKA4_DW$hqdvGlzPZCF&N<{2pNC@I;XT!z@dL`Ox?m?C*oUU9wFoFmkVmw6j46zBr z6Q)L-63h|*r*4>Ml}f0eFS);y^yZ7$zb zHqD}m*VED+N9(eD0aMv6bOm%>6`obs>-6c|W!rL^+-YWD5UcjjvX9XuTTG|N(bE~s zY?*v9(-*6OzRjnLnJe#V{?3aJX?Agvu15PTtBUJ+p4JdA`<`ZhEYj&()JvVX>8kr} zgUq*&$Ghm&le6c)j~8Vzp5{hR$7Pacla2g3dtcEMEwhk}-aiJ=F|3S-BB~2>nO4qJ zi=tMUo>{>_Ob(bw(==OLMYp|}~ZU%O&d`_P}gTHgN*9KfCTKjs{T_y1=B z?XjZFXU?ZnHzcF#s_IejhNu?(BSaegQ;9GWk^uk1k^%oah-dnf`~OSb59;qIe7(zG zH`ErA0fB%(Kp-IS10x`Tomqxs=_#?)VM>uVf|g`oH}@hGqEvA7a_vM4zEG_k6lxu01ieEmm8Tuq_f>)7GzL{+DT4Awd*?=uU+9~{v|i>P=K#&uiM+@U;@A zWK-|YQww$PqViG`E+)n*2nQ(UF~QghZg6ZIY}gnM4tZiMF-|2q7KxxxOhrs%5o_3v zk+ES6age%_#E2zMb8LiCPFN!or?94oYm+#L23P`NCFhYLLOXb*@)lWU9AjR@MDXbS z?RQ9~4`Hx9Kcnkr-Cp8#YC1#Q$J_H}L;YXsiDs9O8fdgZdpw z>}>zt{eL6~-r4^p2nGNDAggE<|LeL8Y8QF!xdWR-Lj*B9e1_1Z`Gr{Bmtu7r9{Zua z-mZ$@PyB~SN&^2s#0y}X^zP@0?eU-H{Kr=OhX?)dM;SX(E7(VAc0mT0x?G2Wb{d6j zKp-Fx5C{nTa0twFmU>rKUlYf9?Bv_N|D4miGvW0K@#g#o4&VRB!u_Cr5IuRiIyo8j z5(@+-_}b0$Gkvj)EL5&=d1>;w8~XIBK07x|>qC#jGM&w*ZlWNvC?LRqlu9gw1(=sw z!P(Fl!UQZvUO+92oh06y82)^Mq_M(b<&h)A@T5Nf_p&~N);z91Is^0Q?`#}|Gv5F5 zAiW%_aRIt2Q;4LcuH4r%+w7m*|2OE){)Gtkzb|!c$-Rln77MLohn~}sKnoO0(Pka6qA2)sOtjjPA<0F9Apdm%%?PJlu^wTR3f}9JyT$INc2+m0LY2Bo z+kCjL@($GhDlwJ2$Qy9Ytb$~en2A>+IZ9kvg#0`iT_ul-cgg5g^6bIcA**Z}Z~(B(HF&Ar5m|{4vYNJ{c9W@%S2tt-v&Z z<%AK&j!Ak<=oup;l8qP}NRkOgHD&)`*NePv%IXqzyV}xCq&m%{q*>c-Il>!@J(C9~ zC;gC*;g}AZ_f3CTmlIei|JJvyc5m}$Zjs3ZiUxaqxH<5pdka}WC)5Q(m;?gGWhr0tP6t%jdrI6?%$4P|rP|4?pM&h_=5YAJNDFF!+e z=O#b-`rq0AFCRUB@#OT+yZo?a|3~-pb^kM>h>ZLHT{hiS9UXLze0HTul=BrV(0Tsv zApVzZWBq4Da24nOAzp{8ByWT5uCD>cd{a{`qhswgO%cpt?3XzyKzi#9rAlcoiC`%g zUKpNQNsP1tFy!MF(O;Tr$E;^4>mDcoaY|$I0;)H+h~;rvH-(wBfi4O=DJR(Aw)|HO zE&9Nk2Tfz##f!!xI2i>qu3_a~Im1wY^Cnc-b$RowK?Je9KAl3bE@xgkEgUMbY%|AE zEzCJe9!{2s=O}qHG3QG~UzX$91TYj|G+M(-sO_VosV=8w-a?Q?i#70clsq*S#>?(4 zl)6E1->^M)ge&iF&=!2&*P@?*$}3RoyMrrTC;o7bBnK^r^2y(p_)mNk|Fby_6#1V| zT6A|6H-yKWXV{(GRY_a?k3>Mb{}ZMtG5}Qf2&I}M{yRuy`jg|oIoZYf?JkPDcm3 z?))!NY%~7@`QP~c=OFLQlK<`c8X%c=%h}RQ6HMD^GdBokZnS^%#EZz(ejsi)nA)#M zAa-^9-uOQf`JY2905|;KJOOGyBW~ya)JOaurVB>?Z$Iu>liFYw-8=)Xe_`qy8gQ*i zEJO=L3q%V<3;ci&PqIn?_abld!Ax4Gf-TyUKK3XcKB_mIfGV=884#+q6MM_z9S3#2P@x!egFUf literal 15360 zcmeHOTW{Mo6wdR0g{Qs7Ht(bi?4fH4v_P>GZPo!Rih$xJ(P~SEBo_&S{P!Knj$?am z)kLz<3k3t>^@#iqk90mB8hfg2qL?y9Ag3+2)O%%-k1=6b3(TYz+QdS}i7?nu&ZWUT z<^=Ir$#{I!ej&t=cx7{l9(o`pt{eKfg3wgb@-9^t!xu?@&s&^-mBifTKZYoqTCfcXdCz{znHv`3~)H z|HcsLZ2$k3dDb$g7W>B>i(UH{6bJinfwhBff zwKA4_DW$hqdvGlzPZCF&N<{2pNC@I;XT!z@dL`Ox?m?C*oUU9wFoFmkVmw6j46zBr z6Q)L-63h|*r*4>Ml}f0eFS);y^yZ7$zb zHqD}m*VED+N9(eD0aMv6bOm%>6`obs>-6c|W!rL^+-YWD5UcjjvX9XuTTG|N(bE~s zY?*v9(-*6OzRjnLnJe#V{?3aJX?Agvu15PTtBUJ+p4JdA`<`ZhEYj&()JvVX>8kr} zgUq*&$Ghm&le6c)j~8Vzp5{hR$7Pacla2g3dtcEMEwhk}-aiJ=F|3S-BB~2>nO4qJ zi=tMUo>{>_Ob(bw(==OLMYp|}~ZU%O&d`_P}gTHgN*9KfCTKjs{T_y1=B z?XjZFXU?ZnHzcF#s_IejhNu?(BSaegQ;9GWk^uk1k^%oah-dnf`~OSb59;qIe7(zG zH`ErA0fB%(Kp-IS10x`Tomqxs=_#?)VM>uVf|g`oH}@hGqEvA7a_vM4zEG_k6lxu01ieEmm8Tuq_f>)7GzL{+DT4Awd*?=uU+9~{v|i>P=K#&uiM+@U;@A zWK-|YQww$PqViG`E+)n*2nQ(UF~QghZg6ZIY}gnM4tZiMF-|2q7KxxxOhrs%5o_3v zk+ES6age%_#E2zMb8LiCPFN!or?94oYm+#L23P`NCFhYLLOXb*@)lWU9AjR@MDXbS z?RQ9~4`Hx9Kcnkr-Cp8#YC1#Q$J_H}L;YXsiDs9O8fdgZdpw z>}>zt{eL6~-r4^p2nGNDAggE<|LeL8Y8QF!xdWR-Lj*B9e1_1Z`Gr{Bmtu7r9{Zua z-mZ$@PyB~SN&^2s#0y}X^zP@0?eU-H{Kr=OhX?)dM;SX(E7(VAc0mT0x?G2Wb{d6j zKp-Fx5C{nTa0twFmU>rKUlYf9?Bv_N|D4miGvW0K@#g#o4&VRB!u_Cr5IuRiIyo8j z5(@+-_}b0$Gkvj)EL5&=d1>;w8~XIBK07x|>qC#jGM&w*ZlWNvC?LRqlu9gw1(=sw z!P(Fl!UQZvUO+92oh06y82)^Mq_M(b<&h)A@T5Nf_p&~N);z91Is^0Q?`#}|Gv5F5 zAiW%_aRIt2Q;4LcuH4r%+w7m*|2OE){)Gtkzb|!c$-Rln77MLohn~}sKnoO0(Pka6qA2)sOtjjPA<0F9Apdm%%?PJlu^wTR3f}9JyT$INc2+m0LY2Bo z+kCjL@($GhDlwJ2$Qy9Ytb$~en2A>+IZ9kvg#0`iT_ul-cgg5g^6bIcA**Z}Z~(B(HF&Ar5m|{4vYNJ{c9W@%S2tt-v&Z z<%AK&j!Ak<=oup;l8qP}NRkOgHD&)`*NePv%IXqzyV}xCq&m%{q*>c-Il>!@J(C9~ zC;gC*;g}AZ_f3CTmlIei|JJvyc5m}$Zjs3ZiUxaqxH<5pdka}WC)5Q(m;?gGWhr0tP6t%jdrI6?%$4P|rP|4?pM&h_=5YAJNDFF!+e z=O#b-`rq0AFCRUB@#OT+yZo?a|3~-pb^kM>h>ZLHT{hiS9UXLze0HTul=BrV(0Tsv zApVzZWBq4Da24nOAzp{8ByWT5uCD>cd{a{`qhswgO%cpt?3XzyKzi#9rAlcoiC`%g zUKpNQNsP1tFy!MF(O;Tr$E;^4>mDcoaY|$I0;)H+h~;rvH-(wBfi4O=DJR(Aw)|HO zE&9Nk2Tfz##f!!xI2i>qu3_a~Im1wY^Cnc-b$RowK?Je9KAl3bE@xgkEgUMbY%|AE zEzCJe9!{2s=O}qHG3QG~UzX$91TYj|G+M(-sO_VosV=8w-a?Q?i#70clsq*S#>?(4 zl)6E1->^M)ge&iF&=!2&*P@?*$}3RoyMrrTC;o7bBnK^r^2y(p_)mNk|Fby_6#1V| zT6A|6H-yKWXV{(GRY_a?k3>Mb{}ZMtG5}Qf2&I}M{yRuy`jg|oIoZYf?JkPDcm3 z?))!NY%~7@`QP~c=OFLQlK<`c8X%c=%h}RQ6HMD^GdBokZnS^%#EZz(ejsi)nA)#M zAa-^9-uOQf`JY2905|;KJOOGyBW~ya)JOaurVB>?Z$Iu>liFYw-8=)Xe_`qy8gQ*i zEJO=L3q%V<3;ci&PqIn?_abld!Ax4Gf-TyUKK3XcKB_mIfGV=884#+q6MM_z9S3#2P@x!egFUf literal 15360 zcmeHOZExE)5ccQ&3Qzm$7n0(e4D3VJ1ZaU`8QQD^Rs;cG6D_u6NOF-N$ba8Sb`r;3 z;-s?ObfI9tq9`6u&xz#S=^@uxQ^6g#21}temy&VL5jS2amRKPXwGv~VBwSc&xYn3p zPJ~7(mckGIDf6uJ_<=A&oYD@OAFKbEP~5>Z#v+D(1nD4x7!sW>dR**gYG-Ao3Q(!7 zwAF`eD{n{buOd~Lv$S%iniVb@MQUO>je3#EGoPMDqpRp3`x1>lM}O=cz5R7KD~sVc zH)=dAb()Qq%D2V4issV12&d6;FZwGl&eQC4KP{lEA#`(?{WeR-HX3EK@%S3LEu5+x zlp}%&>0{JK>-LQr@+NzDI-re|qc<;5lFcD1eFQd$^R#LZnM_<(L93gK1Dl(3SJVv#Ur==2TdT;j{{O8A zSy5zOex@RTbY6m@<20LHfa*n_R~{~0Mb+iBw(?kAx}t;{n|gneTB!RRm1l}DKB5U{ z%6sK0Q3BT*V=0MY-b&$!)!ZYEg!L>YlqFh8?h~n`wnTZ1tplkyk~lK4gSo!#l+^$` zMI0f_v^3ff!eL6L#3~}aan`_M0BZ!5f-uiG8$JOzgeVe>t( z4fy|ztj_-!K}Zn%{~f#!S5aC!+ihP1@cE{yn0v=YC)#Uj2^L5>t+a675R-rt?h+}5 zjllu5-hjJ-h!zw{9V4YtEE)AmTafjPC;;M=N~M`APVOR>w|QA*YEnD8AZ$OMzzuih zKTB6fA5gPbRmz-wtSkg4Bd^A#Te_Cdz|~*9=n81MxOh}S1hF_joVx73oThY`8Bk(z zPZLMgVNNf4Gg(ADz3AOUoh}spl#gc<2d?=1rhBmxXnT-V#pN_l>kwpq$GzaG7rj^7 zjTg=9Q0g{7klZLqeb{I9?~!u}!uy9<Y2vFPUaDD!df0h4Z;rst~jIkoM0v27vfa{*A@-_{) za#JV<1Ofs9fq=mO4uOfvQt!&@DLnpfefV|jKV$JdE@=Jz4+H;mC(N{0&i@4N{F?<& zB;uRc3j&eg2RF@+)ajg4Ub({MrOBtR@6(HVlR-bNDKGspolM7WBpKq|FpMNH$Sn~R z_MOPYC5Z#taDqH+Mlm#Tf_QHd#sm_^a)+fyjsPiL*SiYd&)MuiVc%Rh1JAKJZUhk7 zkAM2s;j`6t1=y-gVIM{6%58PDZv2xwZg|55hw%@aJi_?jmOYl`UPa}a5nAsWX+20c Zz%abNewia^8O?AW5C{ka1b!R@{sr6MmP`Nu diff --git a/syft/linux/identify_release.go b/syft/linux/identify_release.go index 8b24cae8d88..7678dfbcdd4 100644 --- a/syft/linux/identify_release.go +++ b/syft/linux/identify_release.go @@ -55,30 +55,32 @@ var identityFiles = []parseEntry{ // IdentifyRelease parses distro-specific files to discover and raise linux distribution release details. func IdentifyRelease(resolver source.FileResolver) *Release { + logger := log.Nested("operation", "identify-release") for _, entry := range identityFiles { locations, err := resolver.FilesByPath(entry.path) if err != nil { - log.Warnf("unable to get path locations from %s: %+v", entry.path, err) + logger.WithFields("error", err, "path", entry.path).Trace("unable to get path") continue } for _, location := range locations { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { - log.Debugf("unable to get contents from %s: %s", entry.path, err) + logger.WithFields("error", err, "path", location.RealPath).Trace("unable to get contents") continue } content, err := io.ReadAll(contentReader) internal.CloseAndLogError(contentReader, location.VirtualPath) if err != nil { - log.Warnf("unable to read %q: %+v", location.RealPath, err) - break + logger.WithFields("error", err, "path", location.RealPath).Trace("unable to read contents") + continue } release, err := entry.fn(string(content)) if err != nil { - log.Warnf("unable to parse %q", location.RealPath) + logger.WithFields("error", err, "path", location.RealPath).Trace("unable to parse contents") + continue } if release != nil { diff --git a/syft/pkg/alpm_metadata.go b/syft/pkg/alpm_metadata.go index 567b9fd811c..3e807be9b10 100644 --- a/syft/pkg/alpm_metadata.go +++ b/syft/pkg/alpm_metadata.go @@ -6,11 +6,11 @@ import ( "github.com/scylladb/go-set/strset" - "github.com/anchore/packageurl-go" "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/linux" ) +var _ FileOwner = (*AlpmMetadata)(nil) + const AlpmDBGlob = "**/var/lib/pacman/local/**/desc" type AlpmMetadata struct { @@ -40,34 +40,6 @@ type AlpmFileRecord struct { Digests []file.Digest `mapstructure:"digests" json:"digest,omitempty"` } -// PackageURL returns the PURL for the specific Arch Linux package (see https://github.com/package-url/purl-spec) -func (m AlpmMetadata) PackageURL(distro *linux.Release) string { - qualifiers := map[string]string{ - PURLQualifierArch: m.Architecture, - } - - if m.BasePackage != "" { - qualifiers[PURLQualifierUpstream] = m.BasePackage - } - - distroID := "" - if distro != nil { - distroID = distro.ID - } - - return packageurl.NewPackageURL( - "alpm", - distroID, - m.Package, - m.Version, - purlQualifiers( - qualifiers, - distro, - ), - "", - ).ToString() -} - func (m AlpmMetadata) OwnedFiles() (result []string) { s := strset.New() for _, f := range m.Files { diff --git a/syft/pkg/apk_metadata.go b/syft/pkg/apk_metadata.go index ca888187256..0c8c9ac5f04 100644 --- a/syft/pkg/apk_metadata.go +++ b/syft/pkg/apk_metadata.go @@ -65,7 +65,7 @@ func (m ApkMetadata) PackageURL(distro *linux.Release) string { "", m.Package, m.Version, - purlQualifiers( + PURLQualifiers( qualifiers, distro, ), diff --git a/syft/pkg/cataloger/alpm/cataloger.go b/syft/pkg/cataloger/alpm/cataloger.go index 87a7b285e89..39bc7d816c0 100644 --- a/syft/pkg/cataloger/alpm/cataloger.go +++ b/syft/pkg/cataloger/alpm/cataloger.go @@ -1,48 +1,13 @@ package alpm import ( - "fmt" - - "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/pkg/cataloger/generic" ) const catalogerName = "alpmdb-cataloger" -type Cataloger struct{} - -// NewAlpmdbCataloger returns a new ALPM DB cataloger object. -func NewAlpmdbCataloger() *Cataloger { - return &Cataloger{} -} - -// Name returns a string that uniquely describes a cataloger -func (c *Cataloger) Name() string { - return catalogerName -} - -// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. -func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { - fileMatches, err := resolver.FilesByGlob(pkg.AlpmDBGlob) - if err != nil { - return nil, nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) - } - - var pkgs []pkg.Package - for _, location := range fileMatches { - dbContentReader, err := resolver.FileContentsByLocation(location) - if err != nil { - return nil, nil, err - } - - discoveredPkgs, err := parseAlpmDB(resolver, location.RealPath, dbContentReader) - internal.CloseAndLogError(dbContentReader, location.VirtualPath) - if err != nil { - return nil, nil, fmt.Errorf("unable to catalog package=%+v: %w", location.RealPath, err) - } - pkgs = append(pkgs, discoveredPkgs...) - } - return pkgs, nil, nil +func NewAlpmdbCataloger() *generic.Cataloger { + return generic.NewCataloger(catalogerName). + WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob) } diff --git a/syft/pkg/cataloger/alpm/package.go b/syft/pkg/cataloger/alpm/package.go new file mode 100644 index 00000000000..ac9ac1b6592 --- /dev/null +++ b/syft/pkg/cataloger/alpm/package.go @@ -0,0 +1,52 @@ +package alpm + +import ( + "strings" + + "github.com/anchore/packageurl-go" + "github.com/anchore/syft/syft/linux" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func newPackage(m pkg.AlpmMetadata, release *linux.Release, locations ...source.Location) pkg.Package { + p := pkg.Package{ + Name: m.Package, + Version: m.Version, + Locations: source.NewLocationSet(locations...), + Type: pkg.AlpmPkg, + Licenses: strings.Split(m.License, " "), + PURL: packageURL(m, release), + MetadataType: pkg.AlpmMetadataType, + Metadata: m, + } + p.SetID() + return p +} + +func packageURL(m pkg.AlpmMetadata, distro *linux.Release) string { + if distro == nil || distro.ID != "arch" { + // note: there is no namespace variation (like with debian ID_LIKE for ubuntu ID, for example) + return "" + } + + qualifiers := map[string]string{ + pkg.PURLQualifierArch: m.Architecture, + } + + if m.BasePackage != "" { + qualifiers[pkg.PURLQualifierUpstream] = m.BasePackage + } + + return packageurl.NewPackageURL( + "alpm", // `alpm` for Arch Linux and other users of the libalpm/pacman package manager. (see https://github.com/package-url/purl-spec/pull/164) + distro.ID, + m.Package, + m.Version, + pkg.PURLQualifiers( + qualifiers, + distro, + ), + "", + ).ToString() +} diff --git a/syft/pkg/alpm_metadata_test.go b/syft/pkg/cataloger/alpm/package_test.go similarity index 80% rename from syft/pkg/alpm_metadata_test.go rename to syft/pkg/cataloger/alpm/package_test.go index e31ad02a112..bb88a749cf3 100644 --- a/syft/pkg/alpm_metadata_test.go +++ b/syft/pkg/cataloger/alpm/package_test.go @@ -1,4 +1,4 @@ -package pkg +package alpm import ( "testing" @@ -7,18 +7,32 @@ import ( "github.com/anchore/packageurl-go" "github.com/anchore/syft/syft/linux" + "github.com/anchore/syft/syft/pkg" ) -func TestAlpmMetadata_pURL(t *testing.T) { +func Test_PackageURL(t *testing.T) { tests := []struct { name string - metadata AlpmMetadata + metadata pkg.AlpmMetadata distro linux.Release expected string }{ + { + name: "bad distro id", + metadata: pkg.AlpmMetadata{ + Package: "p", + Version: "v", + Architecture: "a", + }, + distro: linux.Release{ + ID: "something-else", + BuildID: "rolling", + }, + expected: "", + }, { name: "gocase", - metadata: AlpmMetadata{ + metadata: pkg.AlpmMetadata{ Package: "p", Version: "v", Architecture: "a", @@ -31,7 +45,7 @@ func TestAlpmMetadata_pURL(t *testing.T) { }, { name: "missing architecture", - metadata: AlpmMetadata{ + metadata: pkg.AlpmMetadata{ Package: "p", Version: "v", }, @@ -41,7 +55,7 @@ func TestAlpmMetadata_pURL(t *testing.T) { expected: "pkg:alpm/arch/p@v?distro=arch", }, { - metadata: AlpmMetadata{ + metadata: pkg.AlpmMetadata{ Package: "python", Version: "3.10.0", Architecture: "any", @@ -53,7 +67,7 @@ func TestAlpmMetadata_pURL(t *testing.T) { expected: "pkg:alpm/arch/python@3.10.0?arch=any&distro=arch-rolling", }, { - metadata: AlpmMetadata{ + metadata: pkg.AlpmMetadata{ Package: "g plus plus", Version: "v84", Architecture: "x86_64", @@ -66,7 +80,7 @@ func TestAlpmMetadata_pURL(t *testing.T) { }, { name: "add source information as qualifier", - metadata: AlpmMetadata{ + metadata: pkg.AlpmMetadata{ Package: "p", Version: "v", Architecture: "a", @@ -82,12 +96,17 @@ func TestAlpmMetadata_pURL(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - actual := test.metadata.PackageURL(&test.distro) + actual := packageURL(test.metadata, &test.distro) if actual != test.expected { dmp := diffmatchpatch.New() diffs := dmp.DiffMain(test.expected, actual, true) t.Errorf("diff: %s", dmp.DiffPrettyText(diffs)) } + + if test.expected == "" { + return + } + // verify packageurl can parse purl, err := packageurl.FromString(actual) if err != nil { diff --git a/syft/pkg/cataloger/alpm/parse_alpm_db.go b/syft/pkg/cataloger/alpm/parse_alpm_db.go index bf40fb3a3eb..1f376b76674 100644 --- a/syft/pkg/cataloger/alpm/parse_alpm_db.go +++ b/syft/pkg/cataloger/alpm/parse_alpm_db.go @@ -13,11 +13,15 @@ import ( "github.com/mitchellh/mapstructure" "github.com/vbatts/go-mtree" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" "github.com/anchore/syft/syft/source" ) +var _ generic.Parser = parseAlpmDB + var ( ignoredFiles = map[string]bool{ "/set": true, @@ -27,16 +31,50 @@ var ( } ) -func newAlpmDBPackage(d *pkg.AlpmMetadata) *pkg.Package { - return &pkg.Package{ - Name: d.Package, - Version: d.Version, - FoundBy: catalogerName, - Type: "alpm", - Licenses: strings.Split(d.License, " "), - MetadataType: pkg.AlpmMetadataType, - Metadata: *d, +func parseAlpmDB(resolver source.FileResolver, env *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + metadata, err := parseAlpmDBEntry(reader) + if err != nil { + return nil, nil, err + } + + base := filepath.Dir(reader.RealPath) + r, err := getFileReader(filepath.Join(base, "mtree"), resolver) + if err != nil { + return nil, nil, err + } + pkgFiles, err := parseMtree(r) + if err != nil { + return nil, nil, err + } + // The replace the files found the the pacman database with the files from the mtree These contain more metadata and + // thus more useful. + metadata.Files = pkgFiles + + // We only really do this to get any backup database entries from the files database + files := filepath.Join(base, "files") + _, err = getFileReader(files, resolver) + if err != nil { + return nil, nil, err + } + filesMetadata, err := parseAlpmDBEntry(reader) + if err != nil { + return nil, nil, err + } else if filesMetadata != nil { + metadata.Backup = filesMetadata.Backup + } + + return []pkg.Package{ + newPackage(*metadata, env.LinuxRelease, reader.Location), + }, nil, nil +} + +func parseAlpmDBEntry(reader io.Reader) (*pkg.AlpmMetadata, error) { + scanner := newScanner(reader) + metadata, err := parseDatabase(scanner) + if err != nil { + return nil, err } + return metadata, nil } func newScanner(reader io.Reader) *bufio.Scanner { @@ -194,53 +232,3 @@ func parseMtree(r io.Reader) ([]pkg.AlpmFileRecord, error) { } return entries, nil } - -func parseAlpmDBEntry(reader io.Reader) (*pkg.AlpmMetadata, error) { - scanner := newScanner(reader) - metadata, err := parseDatabase(scanner) - if err != nil { - return nil, err - } - if metadata == nil { - return nil, nil - } - return metadata, nil -} - -func parseAlpmDB(resolver source.FileResolver, desc string, reader io.Reader) ([]pkg.Package, error) { - metadata, err := parseAlpmDBEntry(reader) - if err != nil { - return nil, err - } - - base := filepath.Dir(desc) - mtree := filepath.Join(base, "mtree") - r, err := getFileReader(mtree, resolver) - if err != nil { - return nil, err - } - pkgFiles, err := parseMtree(r) - if err != nil { - return nil, err - } - // The replace the files found the the pacman database with the files from the mtree These contain more metadata and - // thus more useful. - metadata.Files = pkgFiles - - // We only really do this to get any backup database entries from the files database - files := filepath.Join(base, "files") - _, err = getFileReader(files, resolver) - if err != nil { - return nil, err - } - filesMetadata, err := parseAlpmDBEntry(reader) - if err != nil { - return nil, err - } else if filesMetadata != nil { - metadata.Backup = filesMetadata.Backup - } - - p := *newAlpmDBPackage(metadata) - p.SetID() - return []pkg.Package{p}, nil -} diff --git a/syft/pkg/cataloger/generic/cataloger.go b/syft/pkg/cataloger/generic/cataloger.go new file mode 100644 index 00000000000..c843c680d26 --- /dev/null +++ b/syft/pkg/cataloger/generic/cataloger.go @@ -0,0 +1,151 @@ +package generic + +import ( + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/linux" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +type processor func(resolver source.FileResolver, env Environment) []request + +type request struct { + source.Location + Parser +} + +// Cataloger implements the Catalog interface and is responsible for dispatching the proper parser function for +// a given path or glob pattern. This is intended to be reusable across many package cataloger types. +type Cataloger struct { + processor []processor + upstreamCataloger string +} + +func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger { + c.processor = append(c.processor, + func(resolver source.FileResolver, env Environment) []request { + var requests []request + for _, g := range globs { + // TODO: add more trace logging here + matches, err := resolver.FilesByGlob(g) + if err != nil { + log.Warnf("unable to process glob=%q: %+v", g, err) + continue + } + requests = append(requests, makeRequests(parser, matches)...) + } + return requests + }, + ) + return c +} + +func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Cataloger { + c.processor = append(c.processor, + func(resolver source.FileResolver, env Environment) []request { + var requests []request + for _, t := range types { + // TODO: add more trace logging here + matches, err := resolver.FilesByMIMEType(t) + if err != nil { + log.Warnf("unable to process mimetype=%q: %+v", t, err) + continue + } + requests = append(requests, makeRequests(parser, matches)...) + } + return requests + }, + ) + return c +} + +func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger { + c.processor = append(c.processor, + func(resolver source.FileResolver, env Environment) []request { + var requests []request + for _, g := range paths { + // TODO: add more trace logging here + matches, err := resolver.FilesByPath(g) + if err != nil { + log.Warnf("unable to process path=%q: %+v", g, err) + continue + } + requests = append(requests, makeRequests(parser, matches)...) + } + return requests + }, + ) + return c +} + +func makeRequests(parser Parser, locations []source.Location) []request { + var requests []request + for _, l := range locations { + requests = append(requests, request{ + Location: l, + Parser: parser, + }) + } + return requests +} + +// NewCataloger if provided path-to-parser-function and glob-to-parser-function lookups creates a Cataloger +func NewCataloger(upstreamCataloger string) *Cataloger { + return &Cataloger{ + upstreamCataloger: upstreamCataloger, + } +} + +// Name returns a string that uniquely describes the upstream cataloger that this Generic Cataloger represents. +func (c *Cataloger) Name() string { + return c.upstreamCataloger +} + +// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { + var packages []pkg.Package + var relationships []artifact.Relationship + + logger := log.Nested("cataloger", c.upstreamCataloger) + + env := Environment{ + // TODO: consider passing into the cataloger, this would affect the cataloger interface (and all implementations). This can be deferred until later. + LinuxRelease: linux.IdentifyRelease(resolver), + } + + for _, req := range c.selectFiles(resolver) { + location, parser := req.Location, req.Parser + + contentReader, err := resolver.FileContentsByLocation(location) + if err != nil { + logger.WithFields("location", location.RealPath, "error", err).Warn("unable to fetch contents") + continue + } + + discoveredPackages, discoveredRelationships, err := parser(resolver, &env, source.NewLocationReadCloser(location, contentReader)) + internal.CloseAndLogError(contentReader, location.VirtualPath) + if err != nil { + logger.WithFields("location", location.RealPath, "error", err).Warnf("cataloger failed") + continue + } + + for _, p := range discoveredPackages { + p.FoundBy = c.upstreamCataloger + packages = append(packages, p) + } + + relationships = append(relationships, discoveredRelationships...) + } + return packages, relationships, nil +} + +// selectFiles takes a set of file trees and resolves and file references of interest for future cataloging +func (c *Cataloger) selectFiles(resolver source.FileResolver) []request { + var requests []request + for _, proc := range c.processor { + requests = append(requests, proc(resolver, Environment{})...) + } + return requests +} diff --git a/syft/pkg/cataloger/generic/cataloger_test.go b/syft/pkg/cataloger/generic/cataloger_test.go new file mode 100644 index 00000000000..fd864787a42 --- /dev/null +++ b/syft/pkg/cataloger/generic/cataloger_test.go @@ -0,0 +1,87 @@ +package generic + +import ( + "fmt" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func Test_Cataloger(t *testing.T) { + allParsedPaths := make(map[string]bool) + parser := func(resolver source.FileResolver, env *Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + allParsedPaths[reader.AccessPath()] = true + contents, err := ioutil.ReadAll(reader) + require.NoError(t, err) + + if len(contents) == 0 { + return nil, nil, nil + } + + p := pkg.Package{ + Name: string(contents), + Locations: source.NewLocationSet(reader.Location), + } + r := artifact.Relationship{ + From: p, + To: p, + Type: artifact.ContainsRelationship, + } + + return []pkg.Package{p}, []artifact.Relationship{r}, nil + } + + upstream := "some-other-cataloger" + + expectedSelection := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt", "test-fixtures/empty.txt"} + resolver := source.NewMockResolverForPaths(expectedSelection...) + cataloger := NewCataloger(upstream). + WithParserByPath(parser, "test-fixtures/another-path.txt", "test-fixtures/last/path.txt"). + WithParserByGlobs(parser, "**/a-path.txt", "**/empty.txt") + + actualPkgs, relationships, err := cataloger.Catalog(resolver) + assert.NoError(t, err) + + expectedPkgs := make(map[string]pkg.Package) + for _, path := range expectedSelection { + require.True(t, allParsedPaths[path]) + if path == "test-fixtures/empty.txt" { + continue // note: empty.txt won't become a package + } + expectedPkgs[path] = pkg.Package{ + FoundBy: upstream, + Name: fmt.Sprintf("%s file contents!", path), + } + } + + assert.Len(t, allParsedPaths, len(expectedSelection)) + assert.Len(t, actualPkgs, len(expectedPkgs)) + assert.Len(t, relationships, len(actualPkgs)) + + for _, p := range actualPkgs { + ls := p.Locations.ToSlice() + require.NotEmpty(t, ls) + ref := ls[0] + exP, ok := expectedPkgs[ref.RealPath] + if !ok { + t.Errorf("missing expected pkg: ref=%+v", ref) + continue + } + + // assigned by the generic cataloger + if p.FoundBy != exP.FoundBy { + t.Errorf("bad upstream: %s", p.FoundBy) + } + + // assigned by the parser + if exP.Name != p.Name { + t.Errorf("bad contents mapping: %+v", p.Locations) + } + } +} diff --git a/syft/pkg/cataloger/generic/parser.go b/syft/pkg/cataloger/generic/parser.go new file mode 100644 index 00000000000..32b62f579aa --- /dev/null +++ b/syft/pkg/cataloger/generic/parser.go @@ -0,0 +1,14 @@ +package generic + +import ( + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/linux" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +type Environment struct { + LinuxRelease *linux.Release +} + +type Parser func(source.FileResolver, *Environment, source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) diff --git a/syft/pkg/cataloger/generic/test-fixtures/a-path.txt b/syft/pkg/cataloger/generic/test-fixtures/a-path.txt new file mode 100644 index 00000000000..67e9540341a --- /dev/null +++ b/syft/pkg/cataloger/generic/test-fixtures/a-path.txt @@ -0,0 +1 @@ +test-fixtures/a-path.txt file contents! \ No newline at end of file diff --git a/syft/pkg/cataloger/generic/test-fixtures/another-path.txt b/syft/pkg/cataloger/generic/test-fixtures/another-path.txt new file mode 100644 index 00000000000..0d654f8fe86 --- /dev/null +++ b/syft/pkg/cataloger/generic/test-fixtures/another-path.txt @@ -0,0 +1 @@ +test-fixtures/another-path.txt file contents! \ No newline at end of file diff --git a/syft/pkg/cataloger/generic/test-fixtures/empty.txt b/syft/pkg/cataloger/generic/test-fixtures/empty.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/syft/pkg/cataloger/generic/test-fixtures/last/path.txt b/syft/pkg/cataloger/generic/test-fixtures/last/path.txt new file mode 100644 index 00000000000..3d4a165ab88 --- /dev/null +++ b/syft/pkg/cataloger/generic/test-fixtures/last/path.txt @@ -0,0 +1 @@ +test-fixtures/last/path.txt file contents! \ No newline at end of file diff --git a/syft/pkg/dpkg_metadata.go b/syft/pkg/dpkg_metadata.go index fb405a85bdd..cf8e6cf0529 100644 --- a/syft/pkg/dpkg_metadata.go +++ b/syft/pkg/dpkg_metadata.go @@ -65,7 +65,7 @@ func (m DpkgMetadata) PackageURL(distro *linux.Release) string { namespace, m.Package, m.Version, - purlQualifiers( + PURLQualifiers( qualifiers, distro, ), diff --git a/syft/pkg/package.go b/syft/pkg/package.go index ccfbbb01f48..2f098164647 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -17,7 +17,7 @@ type Package struct { id artifact.ID `hash:"ignore"` Name string // the package name Version string // the version of the package - FoundBy string `cyclonedx:"foundBy"` // the specific cataloger that discovered this package + FoundBy string `hash:"ignore" cyclonedx:"foundBy"` // the specific cataloger that discovered this package Locations source.LocationSet // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) Licenses []string // licenses discovered with the package metadata Language Language `cyclonedx:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) diff --git a/syft/pkg/package_test.go b/syft/pkg/package_test.go index 7c3246a08ce..732f053ad0a 100644 --- a/syft/pkg/package_test.go +++ b/syft/pkg/package_test.go @@ -190,14 +190,6 @@ func TestIDUniqueness(t *testing.T) { }, expectedIDComparison: assert.NotEqual, }, - { - name: "foundBy is reflected", - transform: func(pkg Package) Package { - pkg.FoundBy = "new!" - return pkg - }, - expectedIDComparison: assert.NotEqual, - }, { name: "metadata mutation is reflected", transform: func(pkg Package) Package { diff --git a/syft/pkg/rpm_metadata.go b/syft/pkg/rpm_metadata.go index f2d8880d630..5e5ed629450 100644 --- a/syft/pkg/rpm_metadata.go +++ b/syft/pkg/rpm_metadata.go @@ -80,7 +80,7 @@ func (m RpmMetadata) PackageURL(distro *linux.Release) string { // for purl the epoch is a qualifier, not part of the version // see https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst under the RPM section fmt.Sprintf("%s-%s", m.Version, m.Release), - purlQualifiers( + PURLQualifiers( qualifiers, distro, ), diff --git a/syft/pkg/url.go b/syft/pkg/url.go index 35b8f9c061a..da4b62c3042 100644 --- a/syft/pkg/url.go +++ b/syft/pkg/url.go @@ -67,7 +67,7 @@ func URL(p Package, release *linux.Release) string { ).ToString() } -func purlQualifiers(vars map[string]string, release *linux.Release) (q packageurl.Qualifiers) { +func PURLQualifiers(vars map[string]string, release *linux.Release) (q packageurl.Qualifiers) { keys := make([]string, 0, len(vars)) for k := range vars { keys = append(keys, k) diff --git a/syft/pkg/url_test.go b/syft/pkg/url_test.go index e2a01af91a4..64da300b185 100644 --- a/syft/pkg/url_test.go +++ b/syft/pkg/url_test.go @@ -200,24 +200,6 @@ func TestPackageURL(t *testing.T) { expected: "pkg:maven/g.id/a@v", }, - { - name: "alpm", - distro: &linux.Release{ - ID: "arch", - BuildID: "rolling", - }, - pkg: Package{ - Name: "linux", - Version: "5.10.0", - Type: AlpmPkg, - Metadata: AlpmMetadata{ - Package: "linux", - Version: "5.10.0", - }, - }, - - expected: "pkg:alpm/arch/linux@5.10.0?distro=arch-rolling", - }, { name: "cocoapods", pkg: Package{ @@ -288,6 +270,7 @@ func TestPackageURL(t *testing.T) { // testing microsoft packages is not valid for purl at this time expectedTypes.Remove(string(KbPkg)) expectedTypes.Remove(string(PortagePkg)) + expectedTypes.Remove(string(AlpmPkg)) for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/syft/source/location.go b/syft/source/location.go index 04acee9f526..f131057d192 100644 --- a/syft/source/location.go +++ b/syft/source/location.go @@ -92,6 +92,13 @@ func NewVirtualLocationFromDirectory(responsePath, virtualResponsePath string, r } } +func (l Location) AccessPath() string { + if l.VirtualPath != "" { + return l.VirtualPath + } + return l.RealPath +} + func (l Location) String() string { str := "" if l.ref.ID() != 0 { diff --git a/syft/source/location_read_closer.go b/syft/source/location_read_closer.go new file mode 100644 index 00000000000..b5aa2b6efb5 --- /dev/null +++ b/syft/source/location_read_closer.go @@ -0,0 +1,15 @@ +package source + +import "io" + +type LocationReadCloser struct { + Location + io.ReadCloser +} + +func NewLocationReadCloser(location Location, reader io.ReadCloser) LocationReadCloser { + return LocationReadCloser{ + Location: location, + ReadCloser: reader, + } +}