Merge pull request #519 from brianlmoon/all_ldjson_data

Load all JSON-LD data

src/LinkedData.php

@@ -4,6 +4,7 @@
 namespace Embed;
 
 use Exception;
+use ML\JsonLD\JsonLD;
 use ML\JsonLD\Document as LdDocument;
 use ML\JsonLD\DocumentInterface;
 use ML\JsonLD\GraphInterface;
@@ -16,7 +17,9 @@ class LinkedData
 
     private ?DocumentInterface $document;
 
-    private function get(string ...$keys)
+    private array $allData;
+
+    public function get(string ...$keys)
     {
         $graph = $this->getGraph();
 
@@ -39,6 +42,15 @@ private function get(string ...$keys)
         return null;
     }
 
+    public function getAll()
+    {
+        if (!isset($this->allData)) {
+            $this->fetchData();
+        }
+
+        return $this->allData;
+    }
+
     private function getGraph(string $name = null): ?GraphInterface
     {
         if (!isset($this->document)) {
@@ -50,20 +62,58 @@ private function getGraph(string $name = null): ?GraphInterface
             }
         }
 
-        return $this->document->getGraph();
+        return $this->document->getGraph($name);
     }
 
     protected function fetchData(): array
     {
+        $this->allData = [];
+
         $document = $this->extractor->getDocument();
-        $content = $document->select('.//script', ['type' => 'application/ld+json'])->str();
+        $nodes = $document->select('.//script', ['type' => 'application/ld+json'])->strAll();
 
-        if (empty($content)) {
+        if (empty($nodes)) {
             return [];
         }
 
         try {
-            return json_decode($content, true) ?: [];
+            $data = [];
+            $request_uri = (string)$this->extractor->getUri();
+            foreach ($nodes as $node) {
+                $ldjson = json_decode($node, true);
+                if (!empty($ldjson)) {
+
+                    // some pages with multiple ld+json blocks will put
+                    // each block into an array (Flickr does this). Most
+                    // appear to put an object in each ld+json block. To
+                    // prevent them from stepping on one another, the ones
+                    // that are not arrays will be put into an array.
+                    if (!array_is_list($ldjson)) {
+                        $ldjson = [$ldjson];
+                    }
+
+                    foreach ($ldjson as $node) {
+                        if (empty($data)) {
+                            $data = $node;
+                        } elseif (isset($node['mainEntityOfPage'])) {
+                            $url = '';
+                            if (is_string($node['mainEntityOfPage'])) {
+                                $url = $node['mainEntityOfPage'];
+                            } elseif (isset($node['mainEntityOfPage']['@id'])) {
+                                $url = $node['mainEntityOfPage']['@id'];
+                            }
+                            if (!empty($url) && $url == $request_uri) {
+                                $data = $node;
+                            }
+                        }
+                    }
+
+
+                    $this->allData = array_merge($this->allData, $ldjson);
+                }
+            }
+
+            return $data;
         } catch (Exception $exception) {
             return [];
         }

src/functions.php

@@ -10,7 +10,7 @@ function clean(string $value, bool $allowHTML = false): ?string
     $value = trim($value);
 
     if (!$allowHTML) {
-        $value = html_entity_decode($value);
+        $value = html_entity_decode($value, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
         $value = strip_tags($value);
     }
 
@@ -154,3 +154,25 @@ function isEmpty(mixed ...$values): bool
 
     return false;
 }
+
+if (!function_exists("array_is_list")) {
+    /**
+     * Polyfil for https://www.php.net/manual/en/function.array-is-list.php
+     * which is only available in PHP 8.1+
+     *
+     * @param      array  $array  The array
+     *
+     * @return     bool
+     */
+    function array_is_list(array $array): bool
+    {
+        $i = -1;
+        foreach ($array as $k => $v) {
+            ++$i;
+            if ($k !== $i) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

tests/PagesTest.php

@@ -188,8 +188,8 @@ public function testSoundCloud()
 
     public function testSpotify()
     {
-        $this->assertEmbed('https://play.spotify.com/album/7s66wU1XJ2NsUuWM2NKiUV');
         $this->assertEmbed('https://open.spotify.com/album/7s66wU1XJ2NsUuWM2NKiUV');
+        $this->assertEmbed('https://play.spotify.com/album/7s66wU1XJ2NsUuWM2NKiUV');
     }
 
     public function testTwitch()

tests/PagesTestCase.php

@@ -107,6 +107,7 @@ private static function getData(Extractor $extractor): array
         if (method_exists($extractor, 'getApi')) {
             $data['api'] = $extractor->getApi()->all();
         }
+        $data['allLinkedData'] = $extractor->getLinkedData()->getAll();
 
         return $data;
     }

tests/cache/consent.google.es.ml.0bcc06470676961770c2d2357693454b.php

@@ -0,0 +1,56 @@
+<?php
+declare(strict_types = 1);
+
+return [
+    'headers' => [
+        'content-type' => [
+            'text/html; charset=utf-8'
+        ],
+        'cache-control' => [
+            'no-cache, no-store, max-age=0, must-revalidate'
+        ],
+        'pragma' => [
+            'no-cache'
+        ],
+        'expires' => [
+            'Mon, 01 Jan 1990 00:00:00 GMT'
+        ],
+        'date' => [
+            'Sun, 04 Apr 2021 15:20:47 GMT'
+        ],
+        'p3p' => [
+            'CP="This is not a P3P policy! See g.co/p3phelp for more info."'
+        ],
+        'content-security-policy' => [
+            'script-src \'nonce-g9/eaYJePAYsVh50Jyl0EQ\' \'unsafe-inline\';object-src \'none\';base-uri \'self\';report-uri /_/IdentityNotFoundHttp/cspreport;worker-src \'self\'',
+            'script-src \'nonce-g9/eaYJePAYsVh50Jyl0EQ\' \'self\' https://apis.google.com https://ssl.gstatic.com https://www.google.com https://www.gstatic.com https://www.google-analytics.com;report-uri /_/IdentityNotFoundHttp/cspreport'
+        ],
+        'content-encoding' => [
+            'gzip'
+        ],
+        'server' => [
+            'ESF'
+        ],
+        'x-xss-protection' => [
+            '0'
+        ],
+        'x-content-type-options' => [
+            'nosniff'
+        ],
+        'set-cookie' => [
+            'NID=212=dAlC8GKROGQ6cWC5EeQ92vga0m4ReROz1kMl9BrboOg7GfaE3zvV7pmmgCmsXsJ7vya8tJGI4jioBfUTai-FbFjJPm264-_PY9-GEu66UJhCsRvBiDJVz3O5Ckjox4e0LsT9RZ2vuLADiJTrbw7nzwn4qwyWUF3duIq6_ZUnLhA; expires=Mon, 04-Oct-2021 15:20:47 GMT; path=/; domain=.google.es; Secure; HttpOnly; SameSite=none'
+        ],
+        'alt-svc' => [
+            'h3-29=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"'
+        ],
+        'Content-Location' => [
+            'https://consent.google.es/\'https:/consent.google.es/ml?continue=https://www.google.es/maps/place/Tordoia,%2BA%2BCoru%25C3%25B1a/@43.0871207,-8.5710004,12z/data%3D!3m1!4b1!4m2!3m1!1s0xd2ef4006f1ef489:0x404f58273ca55a0&gl=ES&hl=es&pc=m&src=1&rffu=true\''
+        ],
+        'X-Request-Time' => [
+            '0.112 ms'
+        ]
+    ],
+    'statusCode' => 404,
+    'reasonPhrase' => 'Not Found',
+    'body' => '<html lang=en><meta charset=utf-8><meta name=viewport content="initial-scale=1, minimum-scale=1, width=device-width"><title>Error 404 (No se ha encontrado.)!!1</title><style nonce="g9/eaYJePAYsVh50Jyl0EQ">*{margin:0;padding:0}html,code{font:15px/22px arial,sans-serif}html{background:#fff;color:#222;padding:15px}body{color:#222;text-align:unset;margin:7% auto 0;max-width:390px;min-height:180px;padding:30px 0 15px;}* > body{background:url(//www.google.com/images/errors/robot.png) 100% 5px no-repeat;padding-right:205px}p{margin:11px 0 22px;overflow:hidden}pre{white-space:pre-wrap;}ins{color:#777;text-decoration:none}a img{border:0}@media screen and (max-width:772px){body{background:none;margin-top:0;max-width:none;padding-right:0}}#logo{background:url(//www.google.com/images/branding/googlelogo/1x/googlelogo_color_150x54dp.png) no-repeat;margin-left:-5px}@media only screen and (min-resolution:192dpi){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat 0% 0%/100% 100%;-moz-border-image:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) 0}}@media only screen and (-webkit-min-device-pixel-ratio:2){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat;-webkit-background-size:100% 100%}}#logo{display:inline-block;height:54px;width:150px}</style><div id="af-error-container"><a href=//www.google.com><span id=logo aria-label=Google></span></a><p><b>404.</b> <ins>Se trata de un error.</ins><p>No se ha encontrado la URL solicitada en este servidor. <ins>Esa es toda la información de la que disponemos.</ins></div>'
+];

tests/cache/consent.youtube.com.ml.96e9b7b22bbf367fbeaa2ae77c21e90d.ph

@@ -0,0 +1,50 @@
+<?php
+declare(strict_types = 1);
+
+return [
+    'headers' => [
+        'content-type' => [
+            'text/html; charset=utf-8'
+        ],
+        'cache-control' => [
+            'no-cache, no-store, max-age=0, must-revalidate'
+        ],
+        'pragma' => [
+            'no-cache'
+        ],
+        'expires' => [
+            'Mon, 01 Jan 1990 00:00:00 GMT'
+        ],
+        'date' => [
+            'Sun, 04 Apr 2021 15:23:17 GMT'
+        ],
+        'content-security-policy' => [
+            'script-src \'nonce-vsx63Xooae1XpA8j6kEGNA\' \'unsafe-inline\';object-src \'none\';base-uri \'self\';report-uri /_/IdentityNotFoundHttp/cspreport;worker-src \'self\'',
+            'script-src \'nonce-vsx63Xooae1XpA8j6kEGNA\' \'self\' https://apis.google.com https://ssl.gstatic.com https://www.google.com https://www.gstatic.com https://www.google-analytics.com;report-uri /_/IdentityNotFoundHttp/cspreport'
+        ],
+        'content-encoding' => [
+            'gzip'
+        ],
+        'server' => [
+            'ESF'
+        ],
+        'x-xss-protection' => [
+            '0'
+        ],
+        'x-content-type-options' => [
+            'nosniff'
+        ],
+        'alt-svc' => [
+            'h3-29=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"'
+        ],
+        'Content-Location' => [
+            'https://consent.youtube.com/\'https:/consent.youtube.com/ml?continue=https://www.youtube.com/channel/UCuZeHD5SGecQomz2pVDHGzg&gl=ES&hl=es&pc=yt&uxe=23983172&src=1&rffu=true\''
+        ],
+        'X-Request-Time' => [
+            '0.127 ms'
+        ]
+    ],
+    'statusCode' => 404,
+    'reasonPhrase' => 'Not Found',
+    'body' => '<html lang=en><meta charset=utf-8><meta name=viewport content="initial-scale=1, minimum-scale=1, width=device-width"><title>Error 404 (No se ha encontrado.)!!1</title><style nonce="vsx63Xooae1XpA8j6kEGNA">*{margin:0;padding:0}html,code{font:15px/22px arial,sans-serif}html{background:#fff;color:#222;padding:15px}body{color:#222;text-align:unset;margin:7% auto 0;max-width:390px;min-height:180px;padding:30px 0 15px;}* > body{background:url(//www.google.com/images/errors/robot.png) 100% 5px no-repeat;padding-right:205px}p{margin:11px 0 22px;overflow:hidden}pre{white-space:pre-wrap;}ins{color:#777;text-decoration:none}a img{border:0}@media screen and (max-width:772px){body{background:none;margin-top:0;max-width:none;padding-right:0}}#logo{background:url(//www.google.com/images/branding/googlelogo/1x/googlelogo_color_150x54dp.png) no-repeat;margin-left:-5px}@media only screen and (min-resolution:192dpi){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat 0% 0%/100% 100%;-moz-border-image:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) 0}}@media only screen and (-webkit-min-device-pixel-ratio:2){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat;-webkit-background-size:100% 100%}}#logo{display:inline-block;height:54px;width:150px}</style><div id="af-error-container"><a href=//www.google.com><span id=logo aria-label=Google></span></a><p><b>404.</b> <ins>Se trata de un error.</ins><p>No se ha encontrado la URL solicitada en este servidor. <ins>Esa es toda la información de la que disponemos.</ins></div>'
+];

tests/cache/consent.youtube.com.ml.96e9b7b22bbf367fbeaa2ae77c21e90d.php

@@ -0,0 +1,50 @@
+<?php
+declare(strict_types = 1);
+
+return [
+    'headers' => [
+        'content-type' => [
+            'text/html; charset=utf-8'
+        ],
+        'cache-control' => [
+            'no-cache, no-store, max-age=0, must-revalidate'
+        ],
+        'pragma' => [
+            'no-cache'
+        ],
+        'expires' => [
+            'Mon, 01 Jan 1990 00:00:00 GMT'
+        ],
+        'date' => [
+            'Sun, 04 Apr 2021 15:23:17 GMT'
+        ],
+        'content-security-policy' => [
+            'script-src \'nonce-vsx63Xooae1XpA8j6kEGNA\' \'unsafe-inline\';object-src \'none\';base-uri \'self\';report-uri /_/IdentityNotFoundHttp/cspreport;worker-src \'self\'',
+            'script-src \'nonce-vsx63Xooae1XpA8j6kEGNA\' \'self\' https://apis.google.com https://ssl.gstatic.com https://www.google.com https://www.gstatic.com https://www.google-analytics.com;report-uri /_/IdentityNotFoundHttp/cspreport'
+        ],
+        'content-encoding' => [
+            'gzip'
+        ],
+        'server' => [
+            'ESF'
+        ],
+        'x-xss-protection' => [
+            '0'
+        ],
+        'x-content-type-options' => [
+            'nosniff'
+        ],
+        'alt-svc' => [
+            'h3-29=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"'
+        ],
+        'Content-Location' => [
+            'https://consent.youtube.com/\'https:/consent.youtube.com/ml?continue=https://www.youtube.com/channel/UCuZeHD5SGecQomz2pVDHGzg&gl=ES&hl=es&pc=yt&uxe=23983172&src=1&rffu=true\''
+        ],
+        'X-Request-Time' => [
+            '0.127 ms'
+        ]
+    ],
+    'statusCode' => 404,
+    'reasonPhrase' => 'Not Found',
+    'body' => '<html lang=en><meta charset=utf-8><meta name=viewport content="initial-scale=1, minimum-scale=1, width=device-width"><title>Error 404 (No se ha encontrado.)!!1</title><style nonce="vsx63Xooae1XpA8j6kEGNA">*{margin:0;padding:0}html,code{font:15px/22px arial,sans-serif}html{background:#fff;color:#222;padding:15px}body{color:#222;text-align:unset;margin:7% auto 0;max-width:390px;min-height:180px;padding:30px 0 15px;}* > body{background:url(//www.google.com/images/errors/robot.png) 100% 5px no-repeat;padding-right:205px}p{margin:11px 0 22px;overflow:hidden}pre{white-space:pre-wrap;}ins{color:#777;text-decoration:none}a img{border:0}@media screen and (max-width:772px){body{background:none;margin-top:0;max-width:none;padding-right:0}}#logo{background:url(//www.google.com/images/branding/googlelogo/1x/googlelogo_color_150x54dp.png) no-repeat;margin-left:-5px}@media only screen and (min-resolution:192dpi){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat 0% 0%/100% 100%;-moz-border-image:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) 0}}@media only screen and (-webkit-min-device-pixel-ratio:2){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat;-webkit-background-size:100% 100%}}#logo{display:inline-block;height:54px;width:150px}</style><div id="af-error-container"><a href=//www.google.com><span id=logo aria-label=Google></span></a><p><b>404.</b> <ins>Se trata de un error.</ins><p>No se ha encontrado la URL solicitada en este servidor. <ins>Esa es toda la información de la que disponemos.</ins></div>'
+];