Merge branch 'main' into feature-unique-hash

mediacloud · Dec 13, 2023 · 611666e · 611666e
2 parents 13e0ffb + 9607cf1
commit 611666e
Show file tree

Hide file tree

Showing 91 changed files with 76,070 additions and 24,267 deletions.
diff --git a/README.md b/README.md
@@ -61,9 +61,7 @@ Distribution
 
 ### Test Cache
 
-Test are run against fixtures by default.
-This can be change with the use of '--use-cache=False' when running tests
-
+Test are run against fixtures by default.  This can be changed with the use of '--use-cache=False' when running tests. 
 When adding new tests, re-run 'scripts/get-test-web-content.py' 
 
 Version History

diff --git a/mcmetadata/test/fixtures/cached-httparchiveorg b/mcmetadata/test/fixtures/cached-httparchiveorg
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><base href="/"><meta charset="utf-8"><link rel="icon" href="/offshoot_assets/favicon.ico"><link rel="preconnect" href="https://analytics.archive.org"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="google-site-verification" content="Q2YSouphkkgHkFNP7FgAkc4TmBs1Gmag3uGNndb53B8"><meta name="google-site-verification" content="bpjKvUvsX0lxfmjg19TLblckWkDpnptZEYsBntApxUk"><meta http-equiv="Pragma" content="no-cache"><meta http-equiv="cache-control" content="no-cache, no-store, must-revalidate"><link rel="stylesheet" href="/offshoot_assets/index.237acf313bce.css"><title>Internet Archive: Digital Library of Free &amp; Borrowable Books, Movies, Music &amp; Wayback Machine</title><script src="https://polyfill.archive.org/v3/polyfill.min.js?features=fetch%2CIntersectionObserver%2CResizeObserver%2CglobalThis%2CElement.prototype.getAttributeNames%2CString.prototype.startsWith%2CArray.prototype.flat%2CURL%2CURLSearchParams"></script><script src="/offshoot_assets/vendor/[email protected]/polyfill-support.js"></script><script src="/offshoot_assets/vendor/@webcomponents/[email protected]/webcomponents-loader.js"></script><script>/* @licstart  The following is the entire license notice for the
+<!DOCTYPE html><html lang="en"><head><base href="/"><meta charset="utf-8"><link rel="icon" href="/offshoot_assets/favicon.ico"><link rel="preconnect" href="https://analytics.archive.org"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="google-site-verification" content="Q2YSouphkkgHkFNP7FgAkc4TmBs1Gmag3uGNndb53B8"><meta name="google-site-verification" content="bpjKvUvsX0lxfmjg19TLblckWkDpnptZEYsBntApxUk"><meta http-equiv="Pragma" content="no-cache"><meta http-equiv="cache-control" content="no-cache, no-store, must-revalidate"><link rel="stylesheet" href="/offshoot_assets/index.34c417fd1d63.css"><title>Internet Archive: Digital Library of Free &amp; Borrowable Books, Movies, Music &amp; Wayback Machine</title><script src="https://polyfill.archive.org/v3/polyfill.min.js?features=fetch%2CIntersectionObserver%2CResizeObserver%2CglobalThis%2CElement.prototype.getAttributeNames%2CString.prototype.startsWith%2CArray.prototype.flat%2CURL%2CURLSearchParams"></script><script src="/offshoot_assets/vendor/[email protected]/polyfill-support.js"></script><script src="/offshoot_assets/vendor/@webcomponents/[email protected]/webcomponents-loader.js"></script><script>/* @licstart  The following is the entire license notice for the
       * JavaScript code in this page.
       *
       * This program is free software: you can redistribute it and/or modify
@@ -16,4 +16,4 @@
       *
       * @licend  The above is the entire license notice
       * for the JavaScript code in this page.
-      */</script></head><body><app-root></app-root><noscript><div class="no-js-redirect-container" data-owner="nginx"><h2>Redirecting you to a lite version of archive.org...</h2></div><meta http-equiv="refresh" data-owner="nginx" content="0; url=/?noscript=true"></noscript><script src="/offshoot_assets/js/webpack-runtime.17ec13d79a7e789f2a1c.js"></script><script src="/offshoot_assets/js/index.9b4b9020657291690f03.js"></script></body></html>
+      */</script></head><body><app-root></app-root><noscript><div class="no-js-redirect-container" data-owner="nginx"><h2>Redirecting you to a lite version of archive.org...</h2></div><meta http-equiv="refresh" data-owner="nginx" content="0; url=/?noscript=true"></noscript><script src="/offshoot_assets/js/webpack-runtime.41ec203f4a5452e1c300.js"></script><script src="/offshoot_assets/js/index.c533971b8ae6b74a763f.js"></script></body></html>
diff --git a/mcmetadata/test/fixtures/cached-httpgpogovfdsyspkgpl b/mcmetadata/test/fixtures/cached-httpgpogovfdsyspkgpl
@@ -40,11 +40,15 @@
     <link rel="apple-touch-icon-precomposed" sizes="180x180"
           href="../lib/bootstrap/ico/apple-touch-icon-180.png">
     <link rel="shortcut icon" href="/favicon.ico">
+    <script src="/app/search/tracker_config.js"></script>
 
 
-    <!-- bundle automatically appended by Webpack -->
-<script src="/app/infrastructure.js"></script>
-<link href="/app/main.min.css" rel="stylesheet"><script type="text/javascript" src="/app/search.bundle.6.15.0.js"></script></head>
+
+        <!-- bundle automatically appended by Webpack -->
+    <script src="/app/infrastructure.js"></script>
+    <script src="/app/search/assets/fr-calendar.js"></script> 
+
+<link href="/app/main.min.css" rel="stylesheet"><script type="text/javascript" src="/app/search.bundle.6.21.0.js"></script></head>
 
 <body>
   <header></header>

diff --git a/mcmetadata/test/fixtures/cached-httpmysitecom b/mcmetadata/test/fixtures/cached-httpmysitecom
@@ -379,7 +379,7 @@ to domain name registration and <a style="color:#184CC3;" href="hosting/cheaphos
     </tr>
   </table>
   <p><div style="font-variant: small-caps; font-weight: bold;">MYSITE IS A UNITED ONLINE COMPANY</div>
-  <div style="margin:5px;">Copyright &copy; 2022 United Online Web Services, Inc. All rights reserved.</div>
+  <div style="margin:5px;">Copyright &copy; 2023 United Online Web Services, Inc. All rights reserved.</div>
   <p></div>
 
             <SCRIPT type="text/javascript" language="javascript"><!--

diff --git a/mcmetadata/test/fixtures/cached-httpnewsmachetecompage2 b/mcmetadata/test/fixtures/cached-httpnewsmachetecompage2
diff --git a/mcmetadata/test/fixtures/cached-httppeoplesdailyngcomfeed b/mcmetadata/test/fixtures/cached-httppeoplesdailyngcomfeed
@@ -0,0 +1 @@
+<html><head><link rel="icon" href="data:;"><meta http-equiv="refresh" content="0;/.well-known/sgcaptcha/?r=%2Ffeed%2F%27&y=ipr:104.28.203.246:1702437203.688"></meta></head><script>(function(){var js = "window['__CF$cv$params']={r:'834b07ea0d614cd4',t:'MTcwMjQzNzIwMy43MDgwMDA='};_cpo=document.createElement('script');_cpo.nonce='',_cpo.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js',document.getElementsByTagName('head')[0].appendChild(_cpo);";var _0xh = document.createElement('iframe');_0xh.height = 1;_0xh.width = 1;_0xh.style.position = 'absolute';_0xh.style.top = 0;_0xh.style.left = 0;_0xh.style.border = 'none';_0xh.style.visibility = 'hidden';document.body.appendChild(_0xh);function handler() {var _0xi = _0xh.contentDocument || _0xh.contentWindow.document;if (_0xi) {var _0xj = _0xi.createElement('script');_0xj.innerHTML = js;_0xi.getElementsByTagName('head')[0].appendChild(_0xj);}}if (document.readyState !== 'loading') {handler();} else if (window.addEventListener) {document.addEventListener('DOMContentLoaded', handler);} else {var prev = document.onreadystatechange || function () {};document.onreadystatechange = function (e) {prev(e);if (document.readyState !== 'loading') {document.onreadystatechange = prev;handler();}};}})();</script></html>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		<html><head><link rel="icon" href="data:;"><meta http-equiv="refresh" content="0;/.well-known/sgcaptcha/?r=%2Ffeed%2F%27&y=ipr:104.28.203.246:1702437203.688"></meta></head><script>(function(){var js = "window['__CF$cv$params']={r:'834b07ea0d614cd4',t:'MTcwMjQzNzIwMy43MDgwMDA='};_cpo=document.createElement('script');_cpo.nonce='',_cpo.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js',document.getElementsByTagName('head')[0].appendChild(_cpo);";var _0xh = document.createElement('iframe');_0xh.height = 1;_0xh.width = 1;_0xh.style.position = 'absolute';_0xh.style.top = 0;_0xh.style.left = 0;_0xh.style.border = 'none';_0xh.style.visibility = 'hidden';document.body.appendChild(_0xh);function handler() {var _0xi = _0xh.contentDocument \|\| _0xh.contentWindow.document;if (_0xi) {var _0xj = _0xi.createElement('script');_0xj.innerHTML = js;_0xi.getElementsByTagName('head')[0].appendChild(_0xj);}}if (document.readyState !== 'loading') {handler();} else if (window.addEventListener) {document.addEventListener('DOMContentLoaded', handler);} else {var prev = document.onreadystatechange \|\| function () {};document.onreadystatechange = function (e) {prev(e);if (document.readyState !== 'loading') {document.onreadystatechange = prev;handler();}};}})();</script></html>