Merge pull request #195 from nyudlts/invenio-220-bigfile

Invenio 220 bigfile

.github/workflows/test.yml

@@ -3,7 +3,7 @@ name: ci:test
 on:
   push:
     branches:
-      - main
+      - invenio-220-bigfile
     paths-ignore:
       - "docs/**"
       - "**.md"
@@ -40,21 +40,34 @@ jobs:
         run: |
           sudo apt-get update -y
           sudo apt-get install -y libxml2-dev libxmlsec1-dev
+      - name: "Check Invenio's requirements"
+        run: |
+          invenio-cli check-requirements
       - name: "Setup mandatory files"
         run: |
           echo -e "[cli]\nproject_dir = "$PWD"\nservices_setup = False" >> .invenio.private
           echo -e FLASK_SECRET_KEY="some random value" >> .env
       - name: "Install project dependencies"
         run: |
-          invenio-cli packages lock --pre --dev
-          invenio-cli install --pre --dev
+          invenio-cli install --dev
+          invenio-cli services setup -N
       - name: Start services and app
         run: |
           docker -v
           docker compose up -d
           #invenio-cli services start
           invenio-cli run &
+      - name: "Check services status"
+        run: |
+          bash check-services.sh
       - name: "Run tests"
         run: |
           pipenv install --dev
-          E2E=no pipenv run pytest -p no:cacheprovider
+          export E2E=yes
+          pipenv run pytest -p no:cacheprovider
+      - name: "Upload screenshots"
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: screenshots
+          path: screenshots/*.png

.gitignore

@@ -42,6 +42,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+screenshots/
 
 # Translations
 *.mo

celerybeat-schedule.bak

@@ -0,0 +1,4 @@
+'entries', (3584, 1633)
+'__version__', (512, 15)
+'tz', (1024, 4)
+'utc_enabled', (1536, 4)

celerybeat-schedule.dir

@@ -0,0 +1,4 @@
+'entries', (3584, 1633)
+'__version__', (512, 15)
+'tz', (1024, 4)
+'utc_enabled', (1536, 4)

check-services.sh

@@ -0,0 +1,23 @@
+# check-services.sh
+#!/bin/bash
+
+# Function to check service status
+check_services() {
+  echo "Checking services status..."
+  status=$(invenio-cli services status)
+  echo "$status"
+
+  if [[ "$status" == *"redis: up and running."* && "$status" == *"postgresql: up and running."* && "$status" == *"search: up and running."* ]]; then
+    echo "All services are up and running."
+    return 0
+  else
+    echo "Some services are not ready yet."
+    return 1
+  fi
+}
+
+# Loop until all services are up
+until check_services; do
+  echo "Waiting for all services to be fully operational..."
+  sleep 10 # wait for 10s
+done

invenio.cfg

@@ -100,14 +100,14 @@ PREVIEWER_PREFERENCE = [
 COMMUNITIES_GROUPS_ENABLED = True
 
 # DOI disabled and not configured
-DATACITE_ENABLED = True 
+DATACITE_ENABLED = True
 
 DATACITE_USERNAME = os.getenv("DATACITE_USERNAME","")
 DATACITE_PASSWORD = os.getenv("DATACITE_PASSWORD","")
 DATACITE_PREFIX = os.getenv("DATACITE_PREFIX","")
-DATACITE_TEST_MODE = os.getenv("DATACITE_TEST_MODE",True)
 
-DATACITE_ENABLED = True
+
+DATACITE_TEST_MODE = os.getenv("DATACITE_TEST_MODE",True)
 
 
 PROPAGATE_EXCEPTIONS = True
@@ -403,3 +403,6 @@ RDM_DOCS_HOMEPAGE       = 'http://inveniordm.docs.cern.ch'
 UV_DEPOSIT_GUIDE        = 'https://guides.nyu.edu/ultraviolet/deposit'
 SCIP_HOMEPAGE           = 'https://library.nyu.edu/departments/scholarly-communications-information-policy/'
 NYU_LIBRARIES_HOMEPAGE  = 'https://library.nyu.edu'
+
+# Max file size for displaying download buttons and link (50 GB)
+MAX_FILE_SIZE = 50 * 1024 * 1024 * 1024

templates/semantic-ui/invenio_app_rdm/records/macros/files.html

@@ -52,12 +52,18 @@
         <th>{{_('Name')}}</th>
         <th>{{_('Size')}}</th>
 	 <th class>
-          {%- if config.RDM_ARCHIVE_DOWNLOAD_ENABLED %}
-          {% set archive_download_url = record.links.archive %}
-          <a role="button" class="ui compact mini button right floated archive-link" href="{{ archive_download_url }}">
-            <i class="file archive icon button"></i> {{_("Download all")}}
-          </a>
-          {%- endif %}
+        {% set ns = namespace(download_all=0) %}
+        {% for file in files %}
+          {% if file.size > config.MAX_FILE_SIZE %}
+            {% set ns.download_all = ns.download_all + 1 %}
+          {% endif %}
+        {% endfor %}
+        {% if config.RDM_ARCHIVE_DOWNLOAD_ENABLED and ns.download_all == 0%}
+            {% set archive_download_url = record.links.archive %}
+            <a role="button" class="ui compact mini button right floated archive-link" href="{{ archive_download_url }}">
+              <i class="file archive icon"></i> {{_("Download all")}}
+            </a>
+        {% endif %}
         </th>
         <th class="hidden">{{_('Actions')}}</th>
       </tr>
@@ -76,7 +82,11 @@
       <tr>
         <td>
           <div>
-            <a class="wrap-long-link" href="{{ file_url_download }}">{{ file.key }}</a>
+            {% if file.size < config.MAX_FILE_SIZE %}
+                <a class="wrap-long-link" href="{{ file_url_download }}">{{ file.key }}</a>
+            {% else %}
+                {{ file.key }}
+            {% endif %}
           </div>
           <small class="ui text-muted font-tiny">{{ file.checksum }}
           <div class="ui icon inline-block" data-tooltip="{{_('This is the file fingerprint (checksum), which can be used to verify the file integrity.')}}">
@@ -92,10 +102,12 @@
               <i class="eye icon"></i> {{_("Preview")}}
             </a>
             {% endif %}
-            <a role="button" class="ui compact mini button" href="{{ file_url_download }}">
-              <i class="download icon"></i>
-              {{_('Download')}}
-            </a>
+            {% if file.size < config.MAX_FILE_SIZE %}
+              <a role="button" class="ui compact mini button" href="{{ file_url_download }}">
+                <i class="download icon"></i>
+                {{_('Download')}}
+              </a>
+            {% endif %}
           </span>
         </td>
       </tr>

tests/e2e/conftest.py

@@ -18,6 +18,8 @@
 from invenio_config import create_config_loader
 import pytest
 import os
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
 
 
 @pytest.fixture(scope="module")
@@ -37,3 +39,27 @@ def create_app(ultraviolet_instance_path):
         app_class=factory.app_class(),
     )
     return create_app_e2e
+
+@pytest.fixture(scope="module")
+def chrome_driver():
+    chrome_options = Options()
+    chrome_options.add_argument("disable-blink-features=AutomationControlled")
+    chrome_options.add_argument('--headless=new')
+    chrome_options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')
+    chrome_options.add_argument("--window-size=1920,1080")
+    chrome_options.add_argument("--disable-gpu")
+    chrome_options.add_argument("--disable-software-rasterizer")
+    chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
+    chrome_options.page_load_strategy = 'none'
+    chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
+    chrome_options.add_experimental_option('useAutomationExtension', False)
+    chrome_options.add_argument('--ignore-ssl-errors=yes')
+    chrome_options.add_argument('--ignore-certificate-errors')
+    driver = webdriver.Chrome(options=chrome_options)
+    driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
+
+    try:
+        yield driver
+    finally:
+        driver.quit()