diff --git a/docs/contributing/integration-tests.md b/docs/contributing/integration-tests.md new file mode 100644 index 00000000..65e2f60a --- /dev/null +++ b/docs/contributing/integration-tests.md @@ -0,0 +1,16 @@ +# Integration tests + +## Testing most popular datasets + +Some integration tests operate on the most popular collections for each provider in CMR. +Those collection IDs are cached as static data in `tests/integration/popular_collections/` +to give our test suite more stability. The list of most popular collections can be +updated by running a script in the same directory. + +Sometimes, we find collections with unexpected conditions, like 0 granules, and +therefore "comment" those collections from the list by prefixing the line with a `#`. + +!!! note + + Let's consider a CSV format for this data; we may want to, for example, allow + skipping collections with a EULA by representing that as a column. diff --git a/earthaccess/results.py b/earthaccess/results.py index df21e336..e1ab9667 100644 --- a/earthaccess/results.py +++ b/earthaccess/results.py @@ -257,6 +257,9 @@ def _repr_html_(self) -> str: granule_html_repr = _repr_granule_html(self) return granule_html_repr + def __hash__(self) -> int: # type: ignore[override] + return hash(self["meta"]["concept-id"]) + def get_s3_credentials_endpoint(self) -> Union[str, None]: for link in self["umm"]["RelatedUrls"]: if "/s3credentials" in link["URL"]: diff --git a/mkdocs.yml b/mkdocs.yml index 1eb56098..092b2b1d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -60,13 +60,15 @@ nav: - "What is earthaccess?": "index.md" - "Quick start": "quick-start.md" - "Work with us": - - "contributing/index.md" - - "Development": "contributing/development.md" - - "Releasing": "contributing/releasing.md" - - "Our meet-ups": "contributing/our-meet-ups.md" + - "contributing/index.md" # << Link target of the parent node + - "Development Guide": "contributing/development.md" + - "Releasing Guide": "contributing/releasing.md" - "Maintainers Guide": "contributing/maintainers-guide.md" - "Code of Conduct": "contributing/code-of-conduct.md" - - "Contributing naming convention": "contributing/naming-convention.md" + - "Meet-ups": "contributing/our-meet-ups.md" + - "Topics": + - "Naming conventions": "contributing/naming-convention.md" + - "Integration tests": "contributing/integration-tests.md" - "Resources": "resources.md" - USER GUIDE: - "user_guide/index.md" diff --git a/noxfile.py b/noxfile.py index 0f79d18c..d2b890d3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -57,11 +57,6 @@ def integration_tests(session: nox.Session) -> None: EARTHDATA_USERNAME=os.environ["EARTHDATA_USERNAME"], EARTHDATA_PASSWORD=os.environ["EARTHDATA_PASSWORD"], ), - external=True, - # NOTE: integration test are permitted to pass if the failure rate was less than a hardcoded threshold. - # PyTest will return 99 if there were some failures, but less than the threshold. For more details, see: - # `pytest_sessionfinish` in tests/integration/conftest.py - success_codes=[0, 99], ) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1283f43d..603c7e15 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,49 +1,9 @@ import os import pathlib -from warnings import warn import earthaccess import pytest -ACCEPTABLE_FAILURE_RATE = 10 - - -@pytest.hookimpl() -def pytest_sessionfinish(session, exitstatus): - """Return exit code 99 if up to N% of tests have failed. - - N = ACCEPTABLE_FAILURE_RATE - - 99 was chosen arbitrarily to avoid conflict with current and future pytest error - codes (https://docs.pytest.org/en/stable/reference/exit-codes.html), and avoid - other exit codes with special meanings - (https://tldp.org/LDP/abs/html/exitcodes.html). - - IMPORTANT: This is calculated against every test collected in the session, so the - ratio will change depending on which tests are executed! E.g. executing integration - tests and unit tests at the same time allows more tests to fail than executing - integration tests alone. - - NOTE: The return exit code can be customized with the `EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE` - environment variable. - """ - if exitstatus != pytest.ExitCode.TESTS_FAILED: - # Exit status 1 in PyTest indicates "Tests were collected and run but some of - # the tests failed". In all other cases, for example "an internal error happened - # while executing the tests", or "test execution interrupted by the user", we - # want to defer to original pytest behavior. - return - - failure_rate = (100.0 * session.testsfailed) / session.testscollected - if failure_rate <= ACCEPTABLE_FAILURE_RATE: - status_code = os.environ.get("EARTHACCESS_ALLOWABLE_FAILURE_STATUS_CODE", 99) - warn( - f"\nWARNING: The integration test suite has returned {status_code} because the " - "failure rate was less than a hardcoded threshold. For more details see:\n" - "`pytest_sessionfinish` in tests/integration/conftest.py." - ) - session.exitstatus = status_code - @pytest.fixture def mock_missing_netrc(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch): diff --git a/tests/integration/param.py b/tests/integration/param.py new file mode 100644 index 00000000..88b19727 --- /dev/null +++ b/tests/integration/param.py @@ -0,0 +1,17 @@ +from typing import TypedDict + + +class TestParam(TypedDict): + provider_name: str + + # How many of the top collections we will test, e.g. top 3 collections + n_for_top_collections: int + + # How many granules we will query + granules_count: int + + # How many granules we will randomly select from the query + granules_sample_size: int + + # The maximum allowed granule size; if larger we'll try to find another one + granules_max_size_mb: int diff --git a/tests/integration/popular_collections/ASF.txt b/tests/integration/popular_collections/ASF.txt new file mode 100644 index 00000000..e4911cef --- /dev/null +++ b/tests/integration/popular_collections/ASF.txt @@ -0,0 +1,100 @@ +C1214470488-ASF +C1327985661-ASF +C1214470533-ASF +C1206487504-ASF +C1214353986-ASF +C1327985645-ASF +C1595422627-ASF +C1206485527-ASF +C2011599335-ASF +C1206485320-ASF +C1206485940-ASF +C1206487217-ASF +C1214470496-ASF +C1214470576-ASF +C1206156901-ASF +C1214470532-ASF +C1214472977-ASF +C1808440897-ASF +C1214472994-ASF +C1327985660-ASF +C1266376001-ASF +C1206936391-ASF +C1327985617-ASF +C1327985741-ASF +C1214472336-ASF +C1327985579-ASF +C1243122884-ASF +C1327985740-ASF +C1214470732-ASF +C1327985650-ASF +C1206897141-ASF +C1208794942-ASF +C1243124139-ASF +C1327985619-ASF +C1327985646-ASF +C1208662092-ASF +C1214473170-ASF +C1327985739-ASF +C1327985578-ASF +C1206500826-ASF +C1214470561-ASF +C1209373626-ASF +C1207933168-ASF +C1211627521-ASF +C1210197768-ASF +C1214471521-ASF +C1214336717-ASF +C1214336045-ASF +C1214470682-ASF +C1206500991-ASF +C1213921661-ASF +C1327985571-ASF +C2859376221-ASF +C1327985647-ASF +C1214337770-ASF +C1214343609-ASF +C1243124754-ASF +C1214353859-ASF +C1214353754-ASF +C1213928843-ASF +C1214353593-ASF +C1214336154-ASF +C1214354031-ASF +C1661710588-ASF +C1214408428-ASF +C1214419355-ASF +C1661710600-ASF +C1214354144-ASF +C1214354235-ASF +C1214336554-ASF +C1214335430-ASF +C1213925022-ASF +C1213927035-ASF +C1213927939-ASF +C1214335471-ASF +C179001730-ASF +C1213928209-ASF +C1214335903-ASF +C1213926419-ASF +C1661710597-ASF +C1661710603-ASF +C1661710604-ASF +C1243215430-ASF +C1213926777-ASF +C1243168866-ASF +C1214474435-ASF +C1214473367-ASF +C1661710583-ASF +C1661710590-ASF +C1213921626-ASF +C1214473171-ASF +C1214473839-ASF +C1243162394-ASF +C1214473624-ASF +C1243228612-ASF +C1243255360-ASF +C1243140611-ASF +C2795135668-ASF +C2803501758-ASF +C2777443834-ASF diff --git a/tests/integration/popular_collections/GES_DISC.txt b/tests/integration/popular_collections/GES_DISC.txt new file mode 100644 index 00000000..5dd1c1c9 --- /dev/null +++ b/tests/integration/popular_collections/GES_DISC.txt @@ -0,0 +1,100 @@ +C1243477369-GES_DISC +C1692982070-GES_DISC +C2069247359-GES_DISC +C2033151148-GES_DISC +C1569839798-GES_DISC +C2033167496-GES_DISC +C2069246977-GES_DISC +C1235316220-GES_DISC +C1239966757-GES_DISC +C1701805630-GES_DISC +C1442068519-GES_DISC +C1235316219-GES_DISC +C1701805611-GES_DISC +C1701828243-GES_DISC +C1598621095-GES_DISC +C1235316222-GES_DISC +C1235316223-GES_DISC +C1952167462-GES_DISC +C1675477037-GES_DISC +C1266136111-GES_DISC +C1239966818-GES_DISC +C1266136037-GES_DISC +C1281704371-GES_DISC +C1701805652-GES_DISC +C1238517289-GES_DISC +C1266136071-GES_DISC +C1943072252-GES_DISC +C1598621094-GES_DISC +C1266136062-GES_DISC +C2045794784-GES_DISC +C2248663267-GES_DISC +C2623694314-GES_DISC +C1442068516-GES_DISC +C2556149060-GES_DISC +C2467863601-GES_DISC +C2467880659-GES_DISC +C1239966810-GES_DISC +C1238517344-GES_DISC +C1237114212-GES_DISC +C1239966829-GES_DISC +C2011289787-GES_DISC +C1701805672-GES_DISC +C1442068433-GES_DISC +C1266136096-GES_DISC +C1404080675-GES_DISC +C1282032615-GES_DISC +C1342986035-GES_DISC +C1276812899-GES_DISC +C1276812941-GES_DISC +C2756757848-GES_DISC +C1276812926-GES_DISC +C1266136112-GES_DISC +C1266136100-GES_DISC +C2515837343-GES_DISC +C1239966791-GES_DISC +C2569847612-GES_DISC +C1266136072-GES_DISC +C2756347598-GES_DISC +C1488311935-GES_DISC +C1235316199-GES_DISC +C2723754847-GES_DISC +C1729925175-GES_DISC +C1729925806-GES_DISC +C1276812863-GES_DISC +C1235316218-GES_DISC +C2237419562-GES_DISC +C1276812830-GES_DISC +C1276812901-GES_DISC +C1266136114-GES_DISC +C1933574580-GES_DISC +C1243477366-GES_DISC +C1282032565-GES_DISC +C1239966755-GES_DISC +C1943072156-GES_DISC +C1598621098-GES_DISC +C1243477380-GES_DISC +C1282060545-GES_DISC +C1386443916-GES_DISC +C1700900796-GES_DISC +C1266136097-GES_DISC +C1266136113-GES_DISC +C2042565519-GES_DISC +C1266136070-GES_DISC +C1251101497-GES_DISC +C1633993908-GES_DISC +C1239966842-GES_DISC +C1243477371-GES_DISC +C1239966837-GES_DISC +C1251101764-GES_DISC +C1238517305-GES_DISC +C1282060546-GES_DISC +C1223720291-GES_DISC +C1701805601-GES_DISC +C2042566037-GES_DISC +C1701805657-GES_DISC +C1729925154-GES_DISC +C1276812900-GES_DISC +C1239966827-GES_DISC +C1276812893-GES_DISC +C1239536905-GES_DISC diff --git a/tests/integration/popular_collections/GHRC_DAAC.txt b/tests/integration/popular_collections/GHRC_DAAC.txt new file mode 100644 index 00000000..a888ce6d --- /dev/null +++ b/tests/integration/popular_collections/GHRC_DAAC.txt @@ -0,0 +1,100 @@ +C1983762329-GHRC_DAAC +C1996543397-GHRC_DAAC +C1996545162-GHRC_DAAC +C1979566372-GHRC_DAAC +C1996545873-GHRC_DAAC +C1996545587-GHRC_DAAC +C1996545409-GHRC_DAAC +C1979668994-GHRC_DAAC +C1979825245-GHRC_DAAC +C1980470811-GHRC_DAAC +C1996546067-GHRC_DAAC +C2278812167-GHRC_DAAC +C1979701486-GHRC_DAAC +C1995568158-GHRC_DAAC +C1981725006-GHRC_DAAC +C1979562074-GHRC_DAAC +C1983244465-GHRC_DAAC +C1983220658-GHRC_DAAC +C1979643229-GHRC_DAAC +C1979823036-GHRC_DAAC +C1979944726-GHRC_DAAC +C1979827468-GHRC_DAAC +C2428509185-GHRC_DAAC +C2303219035-GHRC_DAAC +C1979870260-GHRC_DAAC +C1979104659-GHRC_DAAC +C1996541017-GHRC_DAAC +C1979956366-GHRC_DAAC +C1979596455-GHRC_DAAC +C2683433889-GHRC_DAAC +C1996546295-GHRC_DAAC +C1979944011-GHRC_DAAC +C1981574739-GHRC_DAAC +C1979952419-GHRC_DAAC +C1981807112-GHRC_DAAC +C1979572853-GHRC_DAAC +C1995863244-GHRC_DAAC +C1979110434-GHRC_DAAC +C2645106424-GHRC_DAAC +C2738393375-GHRC_DAAC +C1981650432-GHRC_DAAC +C1979639569-GHRC_DAAC +C1979103576-GHRC_DAAC +C1995863290-GHRC_DAAC +C1979889849-GHRC_DAAC +C1995564696-GHRC_DAAC +C1979102757-GHRC_DAAC +C1983100289-GHRC_DAAC +C1995865015-GHRC_DAAC +C2023051335-GHRC_DAAC +C1979975136-GHRC_DAAC +C2409563129-GHRC_DAAC +C1979847172-GHRC_DAAC +C1979110748-GHRC_DAAC +C1979816569-GHRC_DAAC +C1983233595-GHRC_DAAC +C1979883245-GHRC_DAAC +C2708951073-GHRC_DAAC +C1995869798-GHRC_DAAC +C2617798948-GHRC_DAAC +C1995565983-GHRC_DAAC +C2004708338-GHRC_DAAC +C1983208537-GHRC_DAAC +C1979098622-GHRC_DAAC +C1995868627-GHRC_DAAC +C1996546500-GHRC_DAAC +C1979094831-GHRC_DAAC +C1979663392-GHRC_DAAC +C1979095265-GHRC_DAAC +C1979882997-GHRC_DAAC +C1979883116-GHRC_DAAC +C1979883359-GHRC_DAAC +C1996546695-GHRC_DAAC +C1979100395-GHRC_DAAC +C1977853903-GHRC_DAAC +C1979096401-GHRC_DAAC +C1979103240-GHRC_DAAC +C1995871269-GHRC_DAAC +C1983255369-GHRC_DAAC +C1979098477-GHRC_DAAC +C1979100261-GHRC_DAAC +C1979102628-GHRC_DAAC +C1979883491-GHRC_DAAC +C2196515446-GHRC_DAAC +C1976770777-GHRC_DAAC +C1979100690-GHRC_DAAC +C1979102483-GHRC_DAAC +C1995871767-GHRC_DAAC +C1995583255-GHRC_DAAC +C1995865470-GHRC_DAAC +C1995871063-GHRC_DAAC +C1979080632-GHRC_DAAC +C1979096645-GHRC_DAAC +C1979097458-GHRC_DAAC +C1979099590-GHRC_DAAC +C1995566252-GHRC_DAAC +C1996546984-GHRC_DAAC +C1979096536-GHRC_DAAC +C1979097179-GHRC_DAAC +C1979112625-GHRC_DAAC diff --git a/tests/integration/popular_collections/LAADS.txt b/tests/integration/popular_collections/LAADS.txt new file mode 100644 index 00000000..44bcd534 --- /dev/null +++ b/tests/integration/popular_collections/LAADS.txt @@ -0,0 +1,100 @@ +C2105092163-LAADS +C1898025206-LAADS +C1897815356-LAADS +C2105091501-LAADS +C2105087643-LAADS +C2771506686-LAADS +C2105092427-LAADS +C2062201748-LAADS +C2105091380-LAADS +C2062213246-LAADS +C2600303218-LAADS +C1593392869-LAADS +C1562021084-LAADS +C1964798938-LAADS +C1286874966-LAADS +C1344465347-LAADS +C2772641628-LAADS +C2600305692-LAADS +C1287064221-LAADS +C2839119522-LAADS +C1625694062-LAADS +C2600305784-LAADS +C1286876651-LAADS +C2837503356-LAADS +C1625667016-LAADS +C2600306111-LAADS +C2839121037-LAADS +C2839119261-LAADS +C2151211533-LAADS +C2839120576-LAADS +C2839117635-LAADS +C1685849802-LAADS +C2230035528-LAADS +C2849305562-LAADS +C2151219110-LAADS +C2751635237-LAADS +C1685833594-LAADS +C2849291562-LAADS +C2001636718-LAADS +C1625657679-LAADS +C2151227530-LAADS +C2738316689-LAADS +C2230044764-LAADS +C2600307564-LAADS +C2596861873-LAADS +C2736308262-LAADS +C2187507677-LAADS +C1443533683-LAADS +C2980666614-LAADS +C2738881722-LAADS +C2738341325-LAADS +C2736301851-LAADS +C1379758607-LAADS +C2739637361-LAADS +C2801308027-LAADS +C1379841358-LAADS +C2737882843-LAADS +C2736729414-LAADS +C2736350355-LAADS +C2736340952-LAADS +C2859273114-LAADS +C1443729298-LAADS +C2737884938-LAADS +C2736726119-LAADS +C2082363819-LAADS +C2751091676-LAADS +C1443535037-LAADS +C1443727145-LAADS +C1379759127-LAADS +C1443528505-LAADS +C2738334103-LAADS +C2736360676-LAADS +C2736284390-LAADS +C1682050863-LAADS +C1378227407-LAADS +C1443533440-LAADS +C1443561895-LAADS +C1379758778-LAADS +C1443536017-LAADS +C2738888256-LAADS +C2738879348-LAADS +C2736723030-LAADS +C2082363925-LAADS +C1443531026-LAADS +C1443775657-LAADS +C1444200390-LAADS +C2738885504-LAADS +C2738349954-LAADS +C2738327749-LAADS +C1682045802-LAADS +C1681179895-LAADS +C2801723593-LAADS +C2082363908-LAADS +C2947062938-LAADS +C2946990777-LAADS +C2947023794-LAADS +C2947036206-LAADS +C2946981699-LAADS +C2946998410-LAADS +C1655784008-LAADS diff --git a/tests/integration/popular_collections/LARC_ASDC.txt b/tests/integration/popular_collections/LARC_ASDC.txt new file mode 100644 index 00000000..a602c92f --- /dev/null +++ b/tests/integration/popular_collections/LARC_ASDC.txt @@ -0,0 +1,100 @@ +C2667982873-LARC_ASDC +C2445512043-LARC_ASDC +C2667982867-LARC_ASDC +C1556717902-LARC_ASDC +C2667982883-LARC_ASDC +C2667982877-LARC_ASDC +C2667982885-LARC_ASDC +C1633034978-LARC_ASDC +C2667982880-LARC_ASDC +C1962643459-LARC_ASDC +C1633036565-LARC_ASDC +C1556717896-LARC_ASDC +C1584977034-LARC_ASDC +C2655162569-LARC_ASDC +C1667168435-LARC_ASDC +C1342627254-LARC_ASDC +C2667982879-LARC_ASDC +C1978623316-LARC_ASDC +C2059838686-LARC_ASDC +C1556717897-LARC_ASDC +C2935447251-LARC_ASDC +C1982417666-LARC_ASDC +C1556717898-LARC_ASDC +C1541534505-LARC_ASDC +C1556717899-LARC_ASDC +C1978624326-LARC_ASDC +C2764847986-LARC_ASDC +C2150640573-LARC_ASDC +C3026778309-LARC_ASDC +C2705848058-LARC_ASDC +C2059838690-LARC_ASDC +C2146747804-LARC_ASDC +C1556717900-LARC_ASDC +C2667982891-LARC_ASDC +C1666134802-LARC_ASDC +C1374321481-LARC_ASDC +C2501666124-LARC_ASDC +C2276362394-LARC_ASDC +C1969999465-LARC_ASDC +C2136445377-LARC_ASDC +C1575511329-LARC_ASDC +C2150427123-LARC_ASDC +C2609869612-LARC_ASDC +C2164793115-LARC_ASDC +C1633036570-LARC_ASDC +C2059838697-LARC_ASDC +C1584977037-LARC_ASDC +C2609962127-LARC_ASDC +C2212314288-LARC_ASDC +C1584977035-LARC_ASDC +C2276336440-LARC_ASDC +C2091727220-LARC_ASDC +C1725575876-LARC_ASDC +C1536056467-LARC_ASDC +C1949868714-LARC_ASDC +C1661297069-LARC_ASDC +C1577484487-LARC_ASDC +C1863115716-LARC_ASDC +C2146773048-LARC_ASDC +C1536056466-LARC_ASDC +C2762334323-LARC_ASDC +C1978623212-LARC_ASDC +C1602408375-LARC_ASDC +C2087541874-LARC_ASDC +C2566353459-LARC_ASDC +C1576365803-LARC_ASDC +C2081907121-LARC_ASDC +C1544804110-LARC_ASDC +C2058673000-LARC_ASDC +C2062255117-LARC_ASDC +C2147305318-LARC_ASDC +C2566393530-LARC_ASDC +C2184128386-LARC_ASDC +C2609920136-LARC_ASDC +C1949868495-LARC_ASDC +C1978624018-LARC_ASDC +C1577484442-LARC_ASDC +C1374321411-LARC_ASDC +C1591855282-LARC_ASDC +C2231134699-LARC_ASDC +C1990753071-LARC_ASDC +C1990753227-LARC_ASDC +C2791474780-LARC_ASDC +C1684217303-LARC_ASDC +C2134682585-LARC_ASDC +C1536049400-LARC_ASDC +C1535917654-LARC_ASDC +C1688111638-LARC_ASDC +C2609947245-LARC_ASDC +C2566342407-LARC_ASDC +C1000000700-LARC_ASDC +C2166631703-LARC_ASDC +C1000000731-LARC_ASDC +C1726753355-LARC_ASDC +C1978623152-LARC_ASDC +C2828848344-LARC_ASDC +C1599922061-LARC_ASDC +C1237207597-LARC_ASDC +C1584977032-LARC_ASDC +C1990752708-LARC_ASDC diff --git a/tests/integration/popular_collections/LARC_CLOUD.txt b/tests/integration/popular_collections/LARC_CLOUD.txt new file mode 100644 index 00000000..5ac6094f --- /dev/null +++ b/tests/integration/popular_collections/LARC_CLOUD.txt @@ -0,0 +1,47 @@ +C2930725014-LARC_CLOUD +C2930730944-LARC_CLOUD +C2724057189-LARC_CLOUD +C2930726639-LARC_CLOUD +C2930760329-LARC_CLOUD +C2732717000-LARC_CLOUD +C2930763263-LARC_CLOUD +C2842845562-LARC_CLOUD +C2724046381-LARC_CLOUD +C2724037909-LARC_CLOUD +C2724036633-LARC_CLOUD +C2930761273-LARC_CLOUD +C2930764281-LARC_CLOUD +C2930759336-LARC_CLOUD +C2930757598-LARC_CLOUD +C2842852230-LARC_CLOUD +C2930766795-LARC_CLOUD +C2930727817-LARC_CLOUD +C2724057249-LARC_CLOUD +C2724037749-LARC_CLOUD +C2842836142-LARC_CLOUD +C2930729926-LARC_CLOUD +C2930728569-LARC_CLOUD +C2724036159-LARC_CLOUD +C2842851180-LARC_CLOUD +C2724035076-LARC_CLOUD +C2724055205-LARC_CLOUD +C2813530054-LARC_CLOUD +C2724056242-LARC_CLOUD +C2813511488-LARC_CLOUD +C2813533459-LARC_CLOUD +C2862468660-LARC_CLOUD +C2862479332-LARC_CLOUD +C2862461566-LARC_CLOUD +C2862455734-LARC_CLOUD +C3094463244-LARC_CLOUD +C2862443109-LARC_CLOUD +C3094428881-LARC_CLOUD +C2862433153-LARC_CLOUD +C2862446411-LARC_CLOUD +C2862454629-LARC_CLOUD +C2724057022-LARC_CLOUD +C2813514891-LARC_CLOUD +C2813523553-LARC_CLOUD +C2813538048-LARC_CLOUD +C2813461507-LARC_CLOUD +C2813542787-LARC_CLOUD diff --git a/tests/integration/popular_collections/LPCLOUD.txt b/tests/integration/popular_collections/LPCLOUD.txt new file mode 100644 index 00000000..c15db7f3 --- /dev/null +++ b/tests/integration/popular_collections/LPCLOUD.txt @@ -0,0 +1,100 @@ +C2763266360-LPCLOUD +C1748058432-LPCLOUD +C2343115666-LPCLOUD +C2565791013-LPCLOUD +C2269056084-LPCLOUD +C1748066515-LPCLOUD +C2218719731-LPCLOUD +C2343116130-LPCLOUD +C1748046084-LPCLOUD +C2524245159-LPCLOUD +C2202498116-LPCLOUD +C2343109950-LPCLOUD +C2484079608-LPCLOUD +C2324689816-LPCLOUD +C2565794007-LPCLOUD +C2271754179-LPCLOUD +C2484080763-LPCLOUD +C2484086411-LPCLOUD +C2278860820-LPCLOUD +C2545303088-LPCLOUD +C2343111356-LPCLOUD +C2343115255-LPCLOUD +C2565805789-LPCLOUD +C2545303093-LPCLOUD +C2343110937-LPCLOUD +C2565805783-LPCLOUD +C2278858993-LPCLOUD +C2565791036-LPCLOUD +C2484080427-LPCLOUD +C2565794059-LPCLOUD +C2343114808-LPCLOUD +C2763264762-LPCLOUD +C2218777082-LPCLOUD +C2565805776-LPCLOUD +C2484086031-LPCLOUD +C2565791021-LPCLOUD +C2269057787-LPCLOUD +C2763268458-LPCLOUD +C2565791029-LPCLOUD +C2565788905-LPCLOUD +C2565788901-LPCLOUD +C2565794061-LPCLOUD +C2565788881-LPCLOUD +C2222147000-LPCLOUD +C2565794824-LPCLOUD +C2343112831-LPCLOUD +C2565794018-LPCLOUD +C2565805799-LPCLOUD +C2763266348-LPCLOUD +C2763266354-LPCLOUD +C2565791040-LPCLOUD +C2327962326-LPCLOUD +C2202497474-LPCLOUD +C2307290656-LPCLOUD +C2565791018-LPCLOUD +C2343113743-LPCLOUD +C2343114343-LPCLOUD +C2759081792-LPCLOUD +C2565794067-LPCLOUD +C2763266352-LPCLOUD +C1711961296-LPCLOUD +C2565794049-LPCLOUD +C2763264719-LPCLOUD +C2565794060-LPCLOUD +C2343113232-LPCLOUD +C2565788885-LPCLOUD +C2327957988-LPCLOUD +C2565794046-LPCLOUD +C2565794796-LPCLOUD +C2565791034-LPCLOUD +C2565791027-LPCLOUD +C2565794064-LPCLOUD +C2763268459-LPCLOUD +C2763266377-LPCLOUD +C2763264735-LPCLOUD +C2763264695-LPCLOUD +C2565794030-LPCLOUD +C2565786756-LPCLOUD +C2763266325-LPCLOUD +C2565791024-LPCLOUD +C2763264768-LPCLOUD +C2565788876-LPCLOUD +C2142771958-LPCLOUD +C2763266356-LPCLOUD +C2763264737-LPCLOUD +C2763261610-LPCLOUD +C2565788888-LPCLOUD +C2484078896-LPCLOUD +C2565788914-LPCLOUD +C2565794038-LPCLOUD +C2565794069-LPCLOUD +C2763264741-LPCLOUD +C2763264722-LPCLOUD +C2565788897-LPCLOUD +C2763264702-LPCLOUD +C2763264710-LPCLOUD +C2565794850-LPCLOUD +C2565794001-LPCLOUD +C2565788912-LPCLOUD +C2565805839-LPCLOUD diff --git a/tests/integration/popular_collections/LPDAAC_ECS.txt b/tests/integration/popular_collections/LPDAAC_ECS.txt new file mode 100644 index 00000000..ff9e9bf6 --- /dev/null +++ b/tests/integration/popular_collections/LPDAAC_ECS.txt @@ -0,0 +1,100 @@ +C1534729776-LPDAAC_ECS +C1000000320-LPDAAC_ECS +C1534584923-LPDAAC_ECS +C1523387372-LPDAAC_ECS +C1632560913-LPDAAC_ECS +C1442270801-LPDAAC_ECS +C1373412034-LPDAAC_ECS +C1412449610-LPDAAC_ECS +C1407099489-LPDAAC_ECS +C1442270800-LPDAAC_ECS +C1632561643-LPDAAC_ECS +C1392010612-LPDAAC_ECS +C1534582789-LPDAAC_ECS +C1412449608-LPDAAC_ECS +C1407099497-LPDAAC_ECS +C1412449609-LPDAAC_ECS +C1407099490-LPDAAC_ECS +C1407099493-LPDAAC_ECS +C1407099488-LPDAAC_ECS +C1373412048-LPDAAC_ECS +C1632561835-LPDAAC_ECS +C1373412073-LPDAAC_ECS +C14758250-LPDAAC_ECS +C1545228916-LPDAAC_ECS +C1534730833-LPDAAC_ECS +C1392010616-LPDAAC_ECS +C1534730469-LPDAAC_ECS +C1553237573-LPDAAC_ECS +C1534731034-LPDAAC_ECS +C1534730762-LPDAAC_ECS +C1392010614-LPDAAC_ECS +C1412449611-LPDAAC_ECS +C1392010615-LPDAAC_ECS +C1410800182-LPDAAC_ECS +C1908348134-LPDAAC_ECS +C1639530606-LPDAAC_ECS +C1639530522-LPDAAC_ECS +C1711966127-LPDAAC_ECS +C1908350066-LPDAAC_ECS +C1575734433-LPDAAC_ECS +C2527691623-LPDAAC_ECS +C2592845930-LPDAAC_ECS +C1908344278-LPDAAC_ECS +C1629256929-LPDAAC_ECS +C2102664483-LPDAAC_ECS +C1392010611-LPDAAC_ECS +C1629257541-LPDAAC_ECS +C2773138594-LPDAAC_ECS +C2600365286-LPDAAC_ECS +C1373412071-LPDAAC_ECS +C1629258322-LPDAAC_ECS +C1392010618-LPDAAC_ECS +C1299783579-LPDAAC_ECS +C1575734501-LPDAAC_ECS +C1632559364-LPDAAC_ECS +C1534730413-LPDAAC_ECS +C1299783608-LPDAAC_ECS +C1299783630-LPDAAC_ECS +C1534730559-LPDAAC_ECS +C2814094878-LPDAAC_ECS +C1575734760-LPDAAC_ECS +C190733714-LPDAAC_ECS +C1299783607-LPDAAC_ECS +C1299783651-LPDAAC_ECS +C1712040022-LPDAAC_ECS +C1299783800-LPDAAC_ECS +C1432078714-LPDAAC_ECS +C179460405-LPDAAC_ECS +C1534582884-LPDAAC_ECS +C1299783609-LPDAAC_ECS +C2801693973-LPDAAC_ECS +C179460406-LPDAAC_ECS +C2052604735-LPDAAC_ECS +C1299783574-LPDAAC_ECS +C2600365285-LPDAAC_ECS +C1534730053-LPDAAC_ECS +C1299783631-LPDAAC_ECS +C1623882456-LPDAAC_ECS +C1299783684-LPDAAC_ECS +C2600365287-LPDAAC_ECS +C1607332989-LPDAAC_ECS +C1607344635-LPDAAC_ECS +C1607329254-LPDAAC_ECS +C1607333589-LPDAAC_ECS +C1607335247-LPDAAC_ECS +C1607336236-LPDAAC_ECS +C1607315042-LPDAAC_ECS +C1607315366-LPDAAC_ECS +C1607317131-LPDAAC_ECS +C1607327798-LPDAAC_ECS +C1607332875-LPDAAC_ECS +C1607332961-LPDAAC_ECS +C1607337482-LPDAAC_ECS +C1607356370-LPDAAC_ECS +C1607357250-LPDAAC_ECS +C1607357378-LPDAAC_ECS +C1632561158-LPDAAC_ECS +C1629257065-LPDAAC_ECS +C1607221735-LPDAAC_ECS +C1607315142-LPDAAC_ECS diff --git a/tests/integration/popular_collections/NSIDC_CPRD.txt b/tests/integration/popular_collections/NSIDC_CPRD.txt new file mode 100644 index 00000000..a6da00d4 --- /dev/null +++ b/tests/integration/popular_collections/NSIDC_CPRD.txt @@ -0,0 +1,37 @@ +C2596864127-NSIDC_CPRD +C2613553260-NSIDC_CPRD +C2670138092-NSIDC_CPRD +C2684928243-NSIDC_CPRD +C2713030505-NSIDC_CPRD +C2547589158-NSIDC_CPRD +C2761722214-NSIDC_CPRD +C2613553243-NSIDC_CPRD +C2776895930-NSIDC_CPRD +C2649212495-NSIDC_CPRD +C2753295020-NSIDC_CPRD +C2752556504-NSIDC_CPRD +C2153551318-NSIDC_CPRD +C2153549818-NSIDC_CPRD +C2613553216-NSIDC_CPRD +C2776895337-NSIDC_CPRD +C2613553327-NSIDC_CPRD +C2753316241-NSIDC_CPRD +C2153547306-NSIDC_CPRD +C2033638023-NSIDC_CPRD +C2769338020-NSIDC_CPRD +C2153549166-NSIDC_CPRD +C2153547514-NSIDC_CPRD +C2153547635-NSIDC_CPRD +C2153549654-NSIDC_CPRD +C2765424272-NSIDC_CPRD +C2769337070-NSIDC_CPRD +C2153549579-NSIDC_CPRD +C2153549738-NSIDC_CPRD +C2153549511-NSIDC_CPRD +C2153549910-NSIDC_CPRD +C2153552369-NSIDC_CPRD +C2754956786-NSIDC_CPRD +C3162179692-NSIDC_CPRD +C3162334027-NSIDC_CPRD +C2153547430-NSIDC_CPRD +C2153549420-NSIDC_CPRD diff --git a/tests/integration/popular_collections/NSIDC_ECS.txt b/tests/integration/popular_collections/NSIDC_ECS.txt new file mode 100644 index 00000000..ab203f09 --- /dev/null +++ b/tests/integration/popular_collections/NSIDC_ECS.txt @@ -0,0 +1,100 @@ +C2559919423-NSIDC_ECS +C2565090645-NSIDC_ECS +C1646610417-NSIDC_ECS +C2564427300-NSIDC_ECS +C1646610079-NSIDC_ECS +C2776463943-NSIDC_ECS +C1646610415-NSIDC_ECS +C2317038246-NSIDC_ECS +C2531308461-NSIDC_ECS +C1600510471-NSIDC_ECS +C2650116584-NSIDC_ECS +C2564625052-NSIDC_ECS +C2399557265-NSIDC_ECS +C1931663473-NSIDC_ECS +C2136471727-NSIDC_ECS +C2541211133-NSIDC_ECS +C2176562253-NSIDC_ECS +C2738530540-NSIDC_ECS +C2776463935-NSIDC_ECS +C2567856357-NSIDC_ECS +C2776464171-NSIDC_ECS +C1646610394-NSIDC_ECS +C1431539277-NSIDC_ECS +C2136471705-NSIDC_ECS +C2537927247-NSIDC_ECS +C2607017115-NSIDC_ECS +C2666857908-NSIDC_ECS +C1431413941-NSIDC_ECS +C1449515322-NSIDC_ECS +C2750966856-NSIDC_ECS +C1000000443-NSIDC_ECS +C1646609734-NSIDC_ECS +C1542606326-NSIDC_ECS +C2623698025-NSIDC_ECS +C1000000461-NSIDC_ECS +C2509060594-NSIDC_ECS +C2560378689-NSIDC_ECS +C2776464127-NSIDC_ECS +C1646610247-NSIDC_ECS +C2561045326-NSIDC_ECS +C2737912334-NSIDC_ECS +C1371883515-NSIDC_ECS +C1000000400-NSIDC_ECS +C2420522159-NSIDC_ECS +C1000000445-NSIDC_ECS +C2737997483-NSIDC_ECS +C1646609754-NSIDC_ECS +C1723866830-NSIDC_ECS +C2794540918-NSIDC_ECS +C1000000460-NSIDC_ECS +C2534576405-NSIDC_ECS +C1646610390-NSIDC_ECS +C189991863-NSIDC_ECS +C1646609808-NSIDC_ECS +C1452437262-NSIDC_ECS +C189991864-NSIDC_ECS +C189991870-NSIDC_ECS +C2776463679-NSIDC_ECS +C2692731693-NSIDC_ECS +C2776463773-NSIDC_ECS +C2548345108-NSIDC_ECS +C2737997243-NSIDC_ECS +C1542606320-NSIDC_ECS +C189991869-NSIDC_ECS +C189991871-NSIDC_ECS +C1631093696-NSIDC_ECS +C1432250096-NSIDC_ECS +C179014688-NSIDC_ECS +C1000000464-NSIDC_ECS +C1601063219-NSIDC_ECS +C1000000420-NSIDC_ECS +C1646610101-NSIDC_ECS +C2253727823-NSIDC_ECS +C2776463920-NSIDC_ECS +C1574621139-NSIDC_ECS +C2559364353-NSIDC_ECS +C1723866745-NSIDC_ECS +C1646610212-NSIDC_ECS +C2519306057-NSIDC_ECS +C179014696-NSIDC_ECS +C2076118670-NSIDC_ECS +C1454773262-NSIDC_ECS +C128599377-NSIDC_ECS +C2817412114-NSIDC_ECS +C2776463734-NSIDC_ECS +C1414573008-NSIDC_ECS +C179014690-NSIDC_ECS +C1532177374-NSIDC_ECS +C1489170961-NSIDC_ECS +C130038008-NSIDC_ECS +C2295286903-NSIDC_ECS +C1646583410-NSIDC_ECS +C1995259960-NSIDC_ECS +C179014697-NSIDC_ECS +C1997893797-NSIDC_ECS +C2537926833-NSIDC_ECS +C2816704526-NSIDC_ECS +C179014698-NSIDC_ECS +C2137003017-NSIDC_ECS +C1442092309-NSIDC_ECS diff --git a/tests/integration/popular_collections/OB_CLOUD.txt b/tests/integration/popular_collections/OB_CLOUD.txt new file mode 100644 index 00000000..84fb872f --- /dev/null +++ b/tests/integration/popular_collections/OB_CLOUD.txt @@ -0,0 +1,35 @@ +C3020920190-OB_CLOUD +C3026581092-OB_CLOUD +C3020923919-OB_CLOUD +C3020920290-OB_CLOUD +C2804798239-OB_CLOUD +C2804798309-OB_CLOUD +C3026580280-OB_CLOUD +C3020924646-OB_CLOUD +C3026580118-OB_CLOUD +C3026586744-OB_CLOUD +C3026581050-OB_CLOUD +C3020922264-OB_CLOUD +C3026579942-OB_CLOUD +C3020920493-OB_CLOUD +C3026581150-OB_CLOUD +C2804798240-OB_CLOUD +C3020920715-OB_CLOUD +C3020923086-OB_CLOUD +C2804798238-OB_CLOUD +C2869693107-OB_CLOUD +C3020922543-OB_CLOUD +C3020922624-OB_CLOUD +C3020922875-OB_CLOUD +C3020923019-OB_CLOUD +C3020924144-OB_CLOUD +C3020924216-OB_CLOUD +C3020924399-OB_CLOUD +C3020924558-OB_CLOUD +C3026586666-OB_CLOUD +C3160685741-OB_CLOUD +C3160685780-OB_CLOUD +C3020918309-OB_CLOUD +C2832273136-OB_CLOUD +C2816780240-OB_CLOUD +C3026586707-OB_CLOUD diff --git a/tests/integration/popular_collections/OB_DAAC.txt b/tests/integration/popular_collections/OB_DAAC.txt new file mode 100644 index 00000000..91850699 --- /dev/null +++ b/tests/integration/popular_collections/OB_DAAC.txt @@ -0,0 +1,100 @@ +C1240264615-OB_DAAC +C1200034467-OB_DAAC +C2954423973-OB_DAAC +C1200034489-OB_DAAC +C1200034493-OB_DAAC +C2954423863-OB_DAAC +C1200034380-OB_DAAC +C2954424200-OB_DAAC +C2954424297-OB_DAAC +C1200034469-OB_DAAC +C1200034342-OB_DAAC +C1200034367-OB_DAAC +C2954424032-OB_DAAC +C2954424154-OB_DAAC +C2954424255-OB_DAAC +C1633360083-OB_DAAC +C1633360084-OB_DAAC +C1633360090-OB_DAAC +C1633360087-OB_DAAC +C1633360089-OB_DAAC +C1633360091-OB_DAAC +C2108358203-OB_DAAC +C1633360095-OB_DAAC +C1633360097-OB_DAAC +C1633360098-OB_DAAC +C1633360099-OB_DAAC +C1633360100-OB_DAAC +C1633360103-OB_DAAC +C1633360101-OB_DAAC +C1633360102-OB_DAAC +C1633360104-OB_DAAC +C1633360105-OB_DAAC +C1633360106-OB_DAAC +C1633360107-OB_DAAC +C2560278694-OB_DAAC +C1633360110-OB_DAAC +C2639478157-OB_DAAC +C1633360111-OB_DAAC +C1633360112-OB_DAAC +C2172083412-OB_DAAC +C1633360085-OB_DAAC +C1633360093-OB_DAAC +C1633360094-OB_DAAC +C3184807056-OB_DAAC +C1633360108-OB_DAAC +C1633360109-OB_DAAC +C2018199934-OB_DAAC +C2077034346-OB_DAAC +C1633360113-OB_DAAC +C1633360126-OB_DAAC +C1633360127-OB_DAAC +C1633360128-OB_DAAC +C1633360129-OB_DAAC +C1633360131-OB_DAAC +C1633360132-OB_DAAC +C1633360133-OB_DAAC +C1633360135-OB_DAAC +C1633360136-OB_DAAC +C1633360137-OB_DAAC +C1633360138-OB_DAAC +C1633360145-OB_DAAC +C1633360156-OB_DAAC +C3065483572-OB_DAAC +C1633360158-OB_DAAC +C1633360159-OB_DAAC +C1633360160-OB_DAAC +C2850171288-OB_DAAC +C1633360162-OB_DAAC +C1633360163-OB_DAAC +C1647028408-OB_DAAC +C1633360165-OB_DAAC +C1633360167-OB_DAAC +C1633360139-OB_DAAC +C1633360140-OB_DAAC +C1633360141-OB_DAAC +C1633360142-OB_DAAC +C1633360144-OB_DAAC +C1633360154-OB_DAAC +C1633360161-OB_DAAC +C1633360164-OB_DAAC +C1633360166-OB_DAAC +C1633360184-OB_DAAC +C1633360169-OB_DAAC +C2896484540-OB_DAAC +C1633360177-OB_DAAC +C2232401497-OB_DAAC +C1633360183-OB_DAAC +C2131352566-OB_DAAC +C1633360190-OB_DAAC +C1633360192-OB_DAAC +C1633360195-OB_DAAC +C1633360196-OB_DAAC +C1633360197-OB_DAAC +C2795192628-OB_DAAC +C1200034503-OB_DAAC +C1200034487-OB_DAAC +C1200034488-OB_DAAC +C1200034490-OB_DAAC +C1633360182-OB_DAAC +C1633360170-OB_DAAC diff --git a/tests/integration/popular_collections/ORNL_CLOUD.txt b/tests/integration/popular_collections/ORNL_CLOUD.txt new file mode 100644 index 00000000..4e5c011b --- /dev/null +++ b/tests/integration/popular_collections/ORNL_CLOUD.txt @@ -0,0 +1,100 @@ +C2237824918-ORNL_CLOUD +C2532426483-ORNL_CLOUD +C2216864285-ORNL_CLOUD +C2236316271-ORNL_CLOUD +C2236316070-ORNL_CLOUD +C2759076389-ORNL_CLOUD +C2773245356-ORNL_CLOUD +C2273976116-ORNL_CLOUD +C2764708636-ORNL_CLOUD +C2274886681-ORNL_CLOUD +C2216863856-ORNL_CLOUD +C2274237497-ORNL_CLOUD +C2273946359-ORNL_CLOUD +C2207986708-ORNL_CLOUD +C2153683336-ORNL_CLOUD +C2236316336-ORNL_CLOUD +C2274733329-ORNL_CLOUD +C2763353619-ORNL_CLOUD +C2111930592-ORNL_CLOUD +C2275408033-ORNL_CLOUD +C2236240052-ORNL_CLOUD +C2274874175-ORNL_CLOUD +C2274853415-ORNL_CLOUD +C2434072484-ORNL_CLOUD +C2706347267-ORNL_CLOUD +C2531982907-ORNL_CLOUD +C2236316372-ORNL_CLOUD +C2389158955-ORNL_CLOUD +C2236223020-ORNL_CLOUD +C2274852550-ORNL_CLOUD +C2764707175-ORNL_CLOUD +C2207946101-ORNL_CLOUD +C2207986936-ORNL_CLOUD +C2170971503-ORNL_CLOUD +C2258632707-ORNL_CLOUD +C2776899492-ORNL_CLOUD +C2236316392-ORNL_CLOUD +C2784899029-ORNL_CLOUD +C2516155224-ORNL_CLOUD +C2274853114-ORNL_CLOUD +C2275408187-ORNL_CLOUD +C2389107206-ORNL_CLOUD +C2556074845-ORNL_CLOUD +C2736724942-ORNL_CLOUD +C2784893549-ORNL_CLOUD +C2216864025-ORNL_CLOUD +C2784886599-ORNL_CLOUD +C2111722183-ORNL_CLOUD +C2111787144-ORNL_CLOUD +C2111720412-ORNL_CLOUD +C2226005584-ORNL_CLOUD +C2706340483-ORNL_CLOUD +C2274742460-ORNL_CLOUD +C2236316143-ORNL_CLOUD +C2785289548-ORNL_CLOUD +C2784899581-ORNL_CLOUD +C2908697274-ORNL_CLOUD +C2706335063-ORNL_CLOUD +C2784893177-ORNL_CLOUD +C2784881406-ORNL_CLOUD +C2216864433-ORNL_CLOUD +C2279583671-ORNL_CLOUD +C2532007210-ORNL_CLOUD +C2785267282-ORNL_CLOUD +C2390408273-ORNL_CLOUD +C2390701035-ORNL_CLOUD +C2784898435-ORNL_CLOUD +C2279583354-ORNL_CLOUD +C2367023363-ORNL_CLOUD +C2389103604-ORNL_CLOUD +C2192631093-ORNL_CLOUD +C2389685421-ORNL_CLOUD +C2908704847-ORNL_CLOUD +C2162145449-ORNL_CLOUD +C2517350332-ORNL_CLOUD +C2111719486-ORNL_CLOUD +C2236316359-ORNL_CLOUD +C2398099021-ORNL_CLOUD +C2531991823-ORNL_CLOUD +C2240727916-ORNL_CLOUD +C2784854847-ORNL_CLOUD +C2706344412-ORNL_CLOUD +C2813390180-ORNL_CLOUD +C2389289428-ORNL_CLOUD +C2143401680-ORNL_CLOUD +C2714723060-ORNL_CLOUD +C2143401689-ORNL_CLOUD +C2236236883-ORNL_CLOUD +C2785293782-ORNL_CLOUD +C2445456434-ORNL_CLOUD +C2784882228-ORNL_CLOUD +C2143403402-ORNL_CLOUD +C2143402675-ORNL_CLOUD +C2162131333-ORNL_CLOUD +C2784891689-ORNL_CLOUD +C2704963648-ORNL_CLOUD +C2180373101-ORNL_CLOUD +C2162118169-ORNL_CLOUD +C2784896705-ORNL_CLOUD +C2143402559-ORNL_CLOUD diff --git a/tests/integration/popular_collections/ORNL_DAAC.txt b/tests/integration/popular_collections/ORNL_DAAC.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/popular_collections/POCLOUD.txt b/tests/integration/popular_collections/POCLOUD.txt new file mode 100644 index 00000000..cd20a249 --- /dev/null +++ b/tests/integration/popular_collections/POCLOUD.txt @@ -0,0 +1,100 @@ +C2799438299-POCLOUD +C1996881146-POCLOUD +# C2204129664-POCLOUD +C2799438306-POCLOUD +C2270392799-POCLOUD +C2075141559-POCLOUD +C2208418228-POCLOUD +C2036877502-POCLOUD +C2036877509-POCLOUD +C1940473819-POCLOUD +C2147480877-POCLOUD +C2147478146-POCLOUD +C2832195379-POCLOUD +C2799438230-POCLOUD +C1940475563-POCLOUD +C2251464384-POCLOUD +C2147485059-POCLOUD +C2147488020-POCLOUD +C1996881636-POCLOUD +C2205121400-POCLOUD +C2357536365-POCLOUD +C2619444006-POCLOUD +C2075141638-POCLOUD +C2205121394-POCLOUD +C2036877535-POCLOUD +C2075141684-POCLOUD +C2146321631-POCLOUD +C2102959417-POCLOUD +C2799438313-POCLOUD +C2706513160-POCLOUD +C2730520815-POCLOUD +C2619443998-POCLOUD +C2075141605-POCLOUD +C2102958977-POCLOUD +C2036880717-POCLOUD +C2251465126-POCLOUD +C2036877465-POCLOUD +C2799438271-POCLOUD +C2599212091-POCLOUD +C2036880657-POCLOUD +C2205556193-POCLOUD +C2205121416-POCLOUD +C2799438266-POCLOUD +C2036881720-POCLOUD +C2596983413-POCLOUD +C2799438335-POCLOUD +C2036880665-POCLOUD +C2205121433-POCLOUD +C2754895884-POCLOUD +C2036882118-POCLOUD +C2183155461-POCLOUD +C2596986276-POCLOUD +C2036881712-POCLOUD +C2832221740-POCLOUD +C1996881456-POCLOUD +C2274919215-POCLOUD +C2208420167-POCLOUD +C2758162595-POCLOUD +C1968979561-POCLOUD +C2754899545-POCLOUD +C1968979597-POCLOUD +C2036881735-POCLOUD +C2799438345-POCLOUD +C2205122298-POCLOUD +C2832196001-POCLOUD +C2746459620-POCLOUD +C2758162616-POCLOUD +C2536962485-POCLOUD +C1968979566-POCLOUD +C2753948997-POCLOUD +C2205553958-POCLOUD +C2036877693-POCLOUD +C2646932894-POCLOUD +C2296989380-POCLOUD +C1968980576-POCLOUD +C2604362899-POCLOUD +C2036882111-POCLOUD +C2296989383-POCLOUD +C2251464495-POCLOUD +C2619443963-POCLOUD +C2586786218-POCLOUD +C2799438351-POCLOUD +C2799438260-POCLOUD +C1968980609-POCLOUD +C2077042612-POCLOUD +C2847232153-POCLOUD +C2706520933-POCLOUD +C2847232536-POCLOUD +C2537038061-POCLOUD +C2491772132-POCLOUD +C2799438350-POCLOUD +C2537039738-POCLOUD +C2617126679-POCLOUD +C2537006834-POCLOUD +C2036877754-POCLOUD +C2727960248-POCLOUD +C2036878243-POCLOUD +C2927907727-POCLOUD +C2257298823-POCLOUD +C2208421887-POCLOUD diff --git a/tests/integration/popular_collections/PODAAC.txt b/tests/integration/popular_collections/PODAAC.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/popular_collections/SEDAC.txt b/tests/integration/popular_collections/SEDAC.txt new file mode 100644 index 00000000..ad55ed38 --- /dev/null +++ b/tests/integration/popular_collections/SEDAC.txt @@ -0,0 +1,100 @@ +C179001766-SEDAC +C179001768-SEDAC +C179001767-SEDAC +C179001764-SEDAC +C179001770-SEDAC +C179001771-SEDAC +C179001852-SEDAC +C179001772-SEDAC +C179001775-SEDAC +C179001776-SEDAC +C179001773-SEDAC +C179001769-SEDAC +C179001774-SEDAC +C179001777-SEDAC +C179001778-SEDAC +C179001781-SEDAC +C179001780-SEDAC +C179001779-SEDAC +C179001783-SEDAC +C179001782-SEDAC +C179001786-SEDAC +C179001784-SEDAC +C179001785-SEDAC +C179001790-SEDAC +C179001788-SEDAC +C179001791-SEDAC +C179001787-SEDAC +C179001794-SEDAC +C179001793-SEDAC +C179001890-SEDAC +C179001891-SEDAC +C179001892-SEDAC +C179001893-SEDAC +C179001894-SEDAC +C179001895-SEDAC +C179001896-SEDAC +C179001897-SEDAC +C179001898-SEDAC +C179001899-SEDAC +C179001948-SEDAC +C1000000060-SEDAC +C1000000061-SEDAC +C179002107-SEDAC +C1000000280-SEDAC +C1000000300-SEDAC +C1000000320-SEDAC +C1000000321-SEDAC +C2556502578-SEDAC +C2848642691-SEDAC +C2187535796-SEDAC +C2848642408-SEDAC +C2091764506-SEDAC +C2848642054-SEDAC +C2302636732-SEDAC +C2673736502-SEDAC +C2563727886-SEDAC +C2673738199-SEDAC +C179001737-SEDAC +C179001903-SEDAC +C179001910-SEDAC +C179001902-SEDAC +C179001904-SEDAC +C179001905-SEDAC +C179001908-SEDAC +C179001909-SEDAC +C179001907-SEDAC +C179001906-SEDAC +C179001901-SEDAC +C1418622315-SEDAC +C1418641019-SEDAC +C179001914-SEDAC +C1418954625-SEDAC +C1000000360-SEDAC +C179001916-SEDAC +C2738394378-SEDAC +C2338359154-SEDAC +C1418651576-SEDAC +C2211120688-SEDAC +C2675647245-SEDAC +C1603439745-SEDAC +C1000000460-SEDAC +C1000000480-SEDAC +C179001815-SEDAC +C179001707-SEDAC +C179002147-SEDAC +C1000000000-SEDAC +C1000000541-SEDAC +C1419908204-SEDAC +C1604900383-SEDAC +C2000613920-SEDAC +C2586824658-SEDAC +C179001887-SEDAC +C1000000220-SEDAC +C179001967-SEDAC +C179001889-SEDAC +C1000000020-SEDAC +C1000000001-SEDAC +C1000000040-SEDAC +C1947982144-SEDAC +C1000000002-SEDAC diff --git a/tests/integration/popular_collections/generate.py b/tests/integration/popular_collections/generate.py new file mode 100644 index 00000000..89ff8276 --- /dev/null +++ b/tests/integration/popular_collections/generate.py @@ -0,0 +1,60 @@ +"""Generate lists of most popular collections for each of the hardcoded DAACs.""" + +from pathlib import Path + +import requests +from earthaccess.daac import DAACS + +THIS_DIR = Path(__file__).parent + +# TODO: Can we query CMR for all providers, instead of relying on hard-coded names in a DAAC list? +# For example, using this URL: "https://cmr.earthdata.nasa.gov/ingest/providers" +all_providers = [ + provider + for daac_info in DAACS + for provider in set( + list(daac_info["cloud-providers"]) + list(daac_info["on-prem-providers"]) + ) +] + + +def top_collections( + *, + provider: str, + num: int = 100, +) -> list[str]: + if num > 2000: + raise RuntimeError( + "Paging not supported, can only get up to 2000 top collections" + ) + + response = requests.post( + "https://cmr.earthdata.nasa.gov/search/collections.json", + data={ + "provider": provider, + "has_granules_or_cwic": True, + "include_facets": "v2", + "include_granule_counts": True, + "include_has_granules": True, + "include_tags": "edsc.*,opensearch.granule.osdd", + "page_num": 1, + "page_size": num, + "sort_key[]": "-usage_score", + }, + ) + collection_ids = [ + collection["id"] for collection in response.json()["feed"]["entry"] + ] + return collection_ids + + +def main(): + for provider in all_providers: + collection_ids = top_collections(provider=provider) + + output = THIS_DIR / f"{provider}.txt" + output.write_text("\n".join(collection_ids)) + + +if __name__ == "__main__": + main() diff --git a/tests/integration/sample.py b/tests/integration/sample.py new file mode 100644 index 00000000..90791d61 --- /dev/null +++ b/tests/integration/sample.py @@ -0,0 +1,68 @@ +import logging +import random +from pathlib import Path +from typing import Union + +logger = logging.getLogger(__name__) + +INTEGRATION_TEST_DIR = Path(__file__).parent +INTEGRATION_TEST_POPULAR_COLLECTIONS_DIR = INTEGRATION_TEST_DIR / "popular_collections" + + +def top_collections_for_provider(provider: str, *, n: int) -> list[str]: + """Return the top collections for this provider. + + Local cache is used as the source for this list. Run + `./popular_collections/generate.py` to refresh it! + + TODO: Skip / exclude collections that have a EULA; filter them out in this function + or use a pytest skip/xfail mark? + """ + popular_collections_file = ( + INTEGRATION_TEST_POPULAR_COLLECTIONS_DIR / f"{provider}.txt" + ) + popular_collections = popular_collections_file.read_text().split("\n") + + return [c for c in popular_collections[:n] if not c.startswith("#")] + + +def get_sample_granules( + granules: list, + sample_size: int, + max_granule_size: Union[int, float], + round_ndigits: int | None = None, +): + """Return a list of randomly-sampled granules and their size in MB. + + Attempt to find only granules smaller or equal to max_granule_size. May return a + sample smaller than sample_size. + """ + granules_set = set(granules) + + sample = [] + total_size = 0 + max_tries = sample_size * 2 + tries = 0 + + while tries <= max_tries: + g = random.sample(list(granules_set), 1)[0] + if g.size() > max_granule_size: + logger.debug( + f"Granule {g['meta']['concept-id']} exceded max size: {g.size()}." + "Trying another random sample." + ) + tries += 1 + continue + else: + logger.debug( + f"Adding granule to random sample: {g['meta']['concept-id']} size: {g.size()}" + ) + + sample.append(g) + granules_set.remove(g) + + total_size += g.size() + if len(sample) >= sample_size: + break + + return sample, round(total_size, round_ndigits) diff --git a/tests/integration/test_cloud_download.py b/tests/integration/test_cloud_download.py index 11fab5a1..b91afffa 100644 --- a/tests/integration/test_cloud_download.py +++ b/tests/integration/test_cloud_download.py @@ -1,52 +1,49 @@ import logging -import random import shutil from pathlib import Path import earthaccess import pytest -from earthaccess import Auth, DataCollections, DataGranules, Store +from earthaccess import Auth, DataGranules, Store + +from .param import TestParam +from .sample import get_sample_granules, top_collections_for_provider logger = logging.getLogger(__name__) -daac_list = [ +daac_list: list[TestParam] = [ { - "short_name": "NSIDC", - "collections_count": 50, - "collections_sample_size": 3, + "provider_name": "NSIDC_CPRD", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "GES_DISC", - "collections_count": 100, - "collections_sample_size": 3, + "provider_name": "GES_DISC", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 150, }, { - "short_name": "PODAAC", - "collections_count": 100, - "collections_sample_size": 3, + "provider_name": "POCLOUD", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "LPDAAC", - "collections_count": 100, - "collections_sample_size": 3, + "provider_name": "LPCLOUD", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "ORNLDAAC", - "collections_count": 100, - "collections_sample_size": 3, + "provider_name": "ORNL_CLOUD", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, @@ -54,55 +51,36 @@ ] -def get_sample_granules(granules, sample_size, max_granule_size): - """Returns a list with sample granules and their size in MB if - the total size is less than the max_granule_size. - """ - files_to_download = [] - total_size = 0 - max_tries = sample_size * 2 - tries = 0 - - while tries <= max_tries: - g = random.sample(granules, 1)[0] - if g.size() > max_granule_size: - tries += 1 - continue - else: - files_to_download.append(g) - total_size += g.size() - if len(files_to_download) >= sample_size: - break - return files_to_download, round(total_size) - - @pytest.mark.parametrize("daac", daac_list) def test_earthaccess_can_download_cloud_collection_granules(tmp_path, daac): """Tests that we can download cloud collections using HTTPS links.""" - daac_shortname = daac["short_name"] - collections_count = daac["collections_count"] - collections_sample_size = daac["collections_sample_size"] + provider = daac["provider_name"] + n_for_top_collections = daac["n_for_top_collections"] + granules_count = daac["granules_count"] granules_sample_size = daac["granules_sample_size"] granules_max_size = daac["granules_max_size_mb"] - collection_query = DataCollections().data_center(daac_shortname).cloud_hosted(True) - hits = collection_query.hits() - logger.info(f"Cloud hosted collections for {daac_shortname}: {hits}") - collections = collection_query.get(collections_count) - assert len(collections) > collections_sample_size - # We sample n cloud hosted collections from the results - random_collections = random.sample(collections, collections_sample_size) + top_collections = top_collections_for_provider( + provider, + n=n_for_top_collections, + ) + logger.info(f"On-premises collections for {provider}: {len(top_collections)}") - for collection in random_collections: - concept_id = collection.concept_id() + for concept_id in top_collections: granule_query = DataGranules().concept_id(concept_id) total_granules = granule_query.hits() granules = granule_query.get(granules_count) - assert isinstance(granules, list) and len(granules) > 0 - assert isinstance(granules[0], earthaccess.DataGranule) + + msg = f"AssertionError for {concept_id}" + assert isinstance(granules, list), msg + assert len(granules) > 0, msg + assert isinstance(granules[0], earthaccess.DataGranule), msg + granules_to_download, total_size_cmr = get_sample_granules( - granules, granules_sample_size, granules_max_size + granules, + granules_sample_size, + granules_max_size, ) if len(granules_to_download) == 0: logger.warning( diff --git a/tests/integration/test_cloud_open.py b/tests/integration/test_cloud_open.py index a0ca5501..e944fd4f 100644 --- a/tests/integration/test_cloud_open.py +++ b/tests/integration/test_cloud_open.py @@ -1,51 +1,48 @@ import logging -import random import earthaccess import magic import pytest -from earthaccess import Auth, DataCollections, DataGranules, Store +from earthaccess import Auth, DataGranules, Store + +from .param import TestParam +from .sample import get_sample_granules, top_collections_for_provider logger = logging.getLogger(__name__) -daacs_list = [ +daacs_list: list[TestParam] = [ { - "short_name": "NSIDC", - "collections_count": 50, - "collections_sample_size": 3, + "provider_name": "NSIDC_CPRD", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "GES_DISC", - "collections_count": 100, - "collections_sample_size": 2, + "provider_name": "GES_DISC", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 130, }, { - "short_name": "PODAAC", - "collections_count": 100, - "collections_sample_size": 2, + "provider_name": "POCLOUD", + "n_for_top_collections": 2, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "LPDAAC", - "collections_count": 100, - "collections_sample_size": 2, + "provider_name": "LPCLOUD", + "n_for_top_collections": 2, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "ORNLDAAC", - "collections_count": 100, - "collections_sample_size": 3, + "provider_name": "ORNL_CLOUD", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 50, @@ -53,53 +50,27 @@ ] -def get_sample_granules(granules, sample_size, max_granule_size): - """Returns a list with sample granules and their size in MB if - the total size is less than the max_granule_size. - """ - files_to_download = [] - total_size = 0 - max_tries = sample_size * 2 - tries = 0 - - while tries <= max_tries: - g = random.sample(granules, 1)[0] - if g.size() > max_granule_size: - tries += 1 - continue - else: - files_to_download.append(g) - total_size += g.size() - if len(files_to_download) >= sample_size: - break - return files_to_download, round(total_size, 2) - - def supported_collection(data_links): return all("podaac-tools.jpl.nasa.gov/drive" not in url for url in data_links) @pytest.mark.parametrize("daac", daacs_list) def test_earthaccess_can_open_onprem_collection_granules(daac): - """Tests that we can download cloud collections using HTTPS links.""" - daac_shortname = daac["short_name"] - collections_count = daac["collections_count"] - collections_sample_size = daac["collections_sample_size"] + """Tests that we can open cloud collections using HTTPS links.""" + provider = daac["provider_name"] + n_for_top_collections = daac["n_for_top_collections"] + granules_count = daac["granules_count"] granules_sample_size = daac["granules_sample_size"] granules_max_size = daac["granules_max_size_mb"] - collection_query = DataCollections().data_center(daac_shortname).cloud_hosted(True) - hits = collection_query.hits() - logger.info(f"Cloud hosted collections for {daac_shortname}: {hits}") - collections = collection_query.get(collections_count) - assert len(collections) > collections_sample_size - # We sample n cloud hosted collections from the results - random_collections = random.sample(collections, collections_sample_size) - logger.info(f"Sampled {len(random_collections)} collections") - - for collection in random_collections: - concept_id = collection.concept_id() + top_collections = top_collections_for_provider( + provider, + n=n_for_top_collections, + ) + logger.info(f"On-premises collections for {provider}: {len(top_collections)}") + + for concept_id in top_collections: granule_query = DataGranules().concept_id(concept_id) total_granules = granule_query.hits() granules = granule_query.get(granules_count) @@ -110,7 +81,10 @@ def test_earthaccess_can_open_onprem_collection_granules(daac): logger.warning(f"PODAAC DRIVE is not supported at the moment: {data_links}") continue granules_to_open, total_size_cmr = get_sample_granules( - granules, granules_sample_size, granules_max_size + granules, + granules_sample_size, + granules_max_size, + round_ndigits=2, ) if len(granules_to_open) == 0: logger.debug( diff --git a/tests/integration/test_onprem_download.py b/tests/integration/test_onprem_download.py index d728be6c..d7ddeb76 100644 --- a/tests/integration/test_onprem_download.py +++ b/tests/integration/test_onprem_download.py @@ -1,35 +1,34 @@ import logging -import random import shutil import earthaccess import pytest -from earthaccess import Auth, DataCollections, DataGranules, Store +from earthaccess import Auth, DataGranules, Store + +from .param import TestParam +from .sample import get_sample_granules, top_collections_for_provider logger = logging.getLogger(__name__) -daacs_list = [ +daacs_list: list[TestParam] = [ { - "short_name": "NSIDC", - "collections_count": 50, - "collections_sample_size": 3, + "provider_name": "NSIDC_ECS", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "GES_DISC", - "collections_count": 100, - "collections_sample_size": 2, + "provider_name": "GES_DISC", + "n_for_top_collections": 2, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 130, }, { - "short_name": "LPDAAC", - "collections_count": 100, - "collections_sample_size": 2, + "provider_name": "LPDAAC_ECS", + "n_for_top_collections": 2, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, @@ -37,53 +36,27 @@ ] -def get_sample_granules(granules, sample_size, max_granule_size): - """Returns a list with sample granules and their size in MB if - the total size is less than the max_granule_size. - """ - files_to_download = [] - total_size = 0 - max_tries = sample_size * 2 - tries = 0 - - while tries <= max_tries: - g = random.sample(granules, 1)[0] - if g.size() > max_granule_size: - tries += 1 - continue - else: - files_to_download.append(g) - total_size += g.size() - if len(files_to_download) >= sample_size: - break - return files_to_download, round(total_size, 2) - - def supported_collection(data_links): return all("podaac-tools.jpl.nasa.gov/drive" not in url for url in data_links) @pytest.mark.parametrize("daac", daacs_list) def test_earthaccess_can_download_onprem_collection_granules(tmp_path, daac): - """Tests that we can download cloud collections using HTTPS links.""" - daac_shortname = daac["short_name"] - collections_count = daac["collections_count"] - collections_sample_size = daac["collections_sample_size"] + """Tests that we can download on-premises collections using HTTPS links.""" + provider = daac["provider_name"] + n_for_top_collections = daac["n_for_top_collections"] + granules_count = daac["granules_count"] granules_sample_size = daac["granules_sample_size"] granules_max_size = daac["granules_max_size_mb"] - collection_query = DataCollections().data_center(daac_shortname).cloud_hosted(False) - hits = collection_query.hits() - logger.info(f"Cloud hosted collections for {daac_shortname}: {hits}") - collections = collection_query.get(collections_count) - assert len(collections) > collections_sample_size - # We sample n cloud hosted collections from the results - random_collections = random.sample(collections, collections_sample_size) - logger.info(f"Sampled {len(random_collections)} collections") + top_collections = top_collections_for_provider( + provider, + n=n_for_top_collections, + ) + logger.info(f"On-premises collections for {provider}: {len(top_collections)}") - for collection in random_collections: - concept_id = collection.concept_id() + for concept_id in top_collections: granule_query = DataGranules().concept_id(concept_id) total_granules = granule_query.hits() granules = granule_query.get(granules_count) @@ -94,7 +67,10 @@ def test_earthaccess_can_download_onprem_collection_granules(tmp_path, daac): logger.warning(f"PODAAC DRIVE is not supported at the moment: {data_links}") continue granules_to_download, total_size_cmr = get_sample_granules( - granules, granules_sample_size, granules_max_size + granules, + granules_sample_size, + granules_max_size, + round_ndigits=2, ) if len(granules_to_download) == 0: logger.debug( diff --git a/tests/integration/test_onprem_open.py b/tests/integration/test_onprem_open.py index dbc41994..ba84b951 100644 --- a/tests/integration/test_onprem_open.py +++ b/tests/integration/test_onprem_open.py @@ -1,35 +1,33 @@ import logging -import random import earthaccess import magic import pytest -from earthaccess import Auth, DataCollections, DataGranules, Store +from earthaccess import Auth, DataGranules, Store -logger = logging.getLogger(__name__) +from .param import TestParam +from .sample import get_sample_granules, top_collections_for_provider +logger = logging.getLogger(__name__) -daacs_list = [ +daacs_list: list[TestParam] = [ { - "short_name": "NSIDC", - "collections_count": 50, - "collections_sample_size": 3, + "provider_name": "NSIDC_ECS", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "LPDAAC", - "collections_count": 100, - "collections_sample_size": 2, + "provider_name": "LPDAAC_ECS", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 100, }, { - "short_name": "GES_DISC", - "collections_count": 100, - "collections_sample_size": 2, + "provider_name": "GES_DISC", + "n_for_top_collections": 3, "granules_count": 100, "granules_sample_size": 2, "granules_max_size_mb": 130, @@ -37,53 +35,27 @@ ] -def get_sample_granules(granules, sample_size, max_granule_size): - """Returns a list with sample granules and their size in MB if - the total size is less than the max_granule_size. - """ - files_to_download = [] - total_size = 0 - max_tries = sample_size * 2 - tries = 0 - - while tries <= max_tries: - g = random.sample(granules, 1)[0] - if g.size() > max_granule_size: - tries += 1 - continue - else: - files_to_download.append(g) - total_size += g.size() - if len(files_to_download) >= sample_size: - break - return files_to_download, round(total_size, 2) - - def supported_collection(data_links): return all("podaac-tools.jpl.nasa.gov/drive" not in url for url in data_links) @pytest.mark.parametrize("daac", daacs_list) def test_earthaccess_can_open_onprem_collection_granules(daac): - """Tests that we can download cloud collections using HTTPS links.""" - daac_shortname = daac["short_name"] - collections_count = daac["collections_count"] - collections_sample_size = daac["collections_sample_size"] + """Tests that we can open on-prem granules using HTTPS links.""" + provider = daac["provider_name"] + n_for_top_collections = daac["n_for_top_collections"] + granules_count = daac["granules_count"] granules_sample_size = daac["granules_sample_size"] granules_max_size = daac["granules_max_size_mb"] - collection_query = DataCollections().data_center(daac_shortname).cloud_hosted(False) - hits = collection_query.hits() - logger.info(f"Cloud hosted collections for {daac_shortname}: {hits}") - collections = collection_query.get(collections_count) - assert len(collections) > collections_sample_size - # We sample n cloud hosted collections from the results - random_collections = random.sample(collections, collections_sample_size) - logger.info(f"Sampled {len(random_collections)} collections") + top_collections = top_collections_for_provider( + provider, + n=n_for_top_collections, + ) + logger.info(f"On-premises collections for {provider}: {len(top_collections)}") - for collection in random_collections: - concept_id = collection.concept_id() + for concept_id in top_collections: granule_query = DataGranules().concept_id(concept_id) total_granules = granule_query.hits() granules = granule_query.get(granules_count) @@ -94,7 +66,10 @@ def test_earthaccess_can_open_onprem_collection_granules(daac): logger.warning(f"PODAAC DRIVE is not supported at the moment: {data_links}") continue granules_to_open, total_size_cmr = get_sample_granules( - granules, granules_sample_size, granules_max_size + granules, + granules_sample_size, + granules_max_size, + round_ndigits=2, ) if len(granules_to_open) == 0: logger.debug(