feat(server): dataset preview routes add head & json-ld data to the h…

…tml `head`

If a request comes to the server whose route matches `/:username/:name` (filtered for any potentially conflicting routes that we have in our app), we request the associated dataset preview from the backend and embed that data into the DOM.

The DOM data is structured in such a way that it can be found by google's dataset search. We've pulled much of that code from our `qri.cloud` repo.

.gitignore

@@ -1,8 +1,7 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
 # dependencies
-/node_modules
-/server/node_modules
+node_modules
 /.pnp
 .pnp.js
 

server/index.js

@@ -1,62 +1,96 @@
 const express = require("express")
 const path = require('path')
+const fs = require('fs')
+const fetch = require('node-fetch')
+const composeHeadData = require('./util').composeHeadData
+const composeJSONLD = require('./util').composeJSONLD
+
+const API_BASE_URL = process.env.REACT_APP_API_BASE_URL || 'http://localhost:2503'
+const PORT = 3000 
+
 const app = express()
-const PORT = 3001 
 
 app.use(express.static(path.join(__dirname, "..", "build")))
 app.use(express.static(path.join(__dirname, "..", "public")))
 
-app.listen(PORT, () => {
+var server = app.listen(PORT, () => {
   console.log(`server started on port ${PORT}`)
 })
 
-// const passThroughRoutes = [
-//   '/login',
-//   '/signup',
-//   '/login/forgot',
-//   '/dashboard',
-//   '/collection',
-//   '/activity',
-//   '/workflow/new',
-//   '/new',
-//   '/run',
-//   '/changes',
-//   '/search',
-//   '/notifications',
-//   '/notification_settings',
-//   '/:username',
-//   '/:username/following'
-// ]
-
-// passThroughRoutes.forEach((route) => {
-//   app.use(route, (req, res, next) => {
-//     console.log(route, "req url", req.url, req.params, req.path, req.query)
-//     res.sendFile(
-//       path.join(__dirname, "..", "build", "index.html")
-//     )
-//   })
-// })
-
-app.use('/:username/:name', async function(req, res) {
-  // Retrieve the ref from our URL path
-  var username = req.params.username
-  var name = req.params.name
-
-  if (username === "ipfs") return
-  console.log("username/name", username, name)
-  res.sendFile(path.join(__dirname, "..", "build", "hello_world.html"))
+// reservedUsernames are a list of words that may be mistaken for usernames
+// during a url regex match of `/:username/:name`, because of how our routes
+// are formed
+// must be updated if routes change, particularly routes with 2 segments as
+// those are the routes at risk for being incorrectly matched
+const reservedUsernames = ['ipfs', 'login', 'workflow'] 
+
+
+// reservedNames are a list of words that may be mistaken for dataset names
+// during a url regex match of `/:username/:name`, because of how our routes
+// are formed
+// must be updated if routes change, particularly routes with 2 segments as
+// those are the routes at risk for being incorrectly matched
+const reservedNames = ['following']
+
+const indexPath = path.join(__dirname, "..", "build", "index.html")
+
+fs.access(indexPath, fs.constants.F_OK, (err) => {
+  if (err !== null) {
+    console.error(`File ${indexPath} does not exist: ${err}.\nApp must be built before attempting to host it.`)
+    server.close(() => {
+      process.exit(1)
+    }) 
+  }
 })
 
-app.use('/', (req, res, next) => {
-  console.log(req.url)
-  res.sendFile(
-    path.join(__dirname, "..", "build", "index.html")
-  ) 
+app.use('/:username/:name', async (req, res) => {
+  const username = req.params.username
+  if (reservedUsernames.some((u) => {
+    return u === username
+  })) {
+    return res.sendFile(indexPath)
+  }
+
+  const name = req.params.name
+  if (reservedNames.some((n) => {
+    return n === name
+  })) {
+    return res.sendFile(indexPath)
+  }
+
+  const datasetPreviewURL = `${API_BASE_URL}/ds/get/${username}/${name}`
+  var dataset = {}
+  try {
+    dataset = await fetch(datasetPreviewURL)
+                      .then(res => res.json())
+                      .then(res => res.data )
+  } catch (e) {
+    console.log(`error fetching dataset ${username}/${name}: ${e}`)
+    return res.sendFile(indexPath)
+  }
+
+  var indexHTML = ""
+  try {
+    indexHTML = fs.readFileSync(indexPath, { encoding: 'utf8' })
+  } catch (e) {
+    console.log(`error reading index.html file: ${e}`)
+    return res.sendFile(indexPath)
+  }
+
+  try {
+    const headData = composeHeadData(dataset)
+    const jld = composeJSONLD(dataset)
+    indexHTML = indexHTML.replace('<head>', headData+jld)
+  } catch (e) {
+    console.log(`error composing dataset ${username/name} data into html tags: ${e}`)
+    return res.sendFile(indexPath)
+  }
+
+  res.contentType('text/html')
+  res.status(200)
+  return res.send(indexHTML)
 })
 
-// app.use('*', (req, res, next) => {
-//   console.log("use *", req.url, req.params, req.path, req.query)
-//   res.sendFile(
-//     path.join(__dirname, "..", "build", "index.html")
-//   )
-// })
+app.use('/', async (req, res) => {
+  return res.sendFile(indexPath)
+})

server/package.json

@@ -4,8 +4,9 @@
   "main": "index.js",
   "dependencies": {
     "express": "4.17.1",
-    "node-fetch": "3.0.0",
-    "nodemon": "2.0.13"
+    "node-fetch": "2.6.1",
+    "nodemon": "2.0.13",
+    "path-to-regexp": "6.2.0"
   },
   "scripts": {
     "dev": "nodemon index.js",

server/util.js

@@ -0,0 +1,50 @@
+const composeHeadData = (dataset) => {
+  const { peername, name, meta } = dataset
+  // start with generic title and description
+  let title = `${peername}/${name} | qri.cloud`
+  let description = `Preview this dataset on qri.cloud`
+
+  // if meta, use meta values
+  if (meta) {
+    if (meta.title) {
+      title = `${meta.title} | qri.cloud`
+    }
+    if (meta.description) {
+      description = `${meta.description}`
+    }
+  }
+
+  const data = { title, description }
+
+  return `<head data=${JSON.stringify(data)}>`
+}
+
+exports.composeHeadData = composeHeadData
+
+const composeJSONLD = (dataset) => {
+  const {
+    peername,
+    name,
+    meta = {}
+  } = dataset
+
+  const jld = {
+    '@context': 'https://schema.org/',
+    '@type': 'Dataset',
+    name: meta.title || name,
+    description: meta.description || `A dataset published on qri.cloud by ${peername}`,
+    url: `https://qri.cloud/${peername}/${name}`,
+    identifier: [`${peername}/${name}`],
+    includedInDataCatalog: {
+      '@type': 'DataCatalog',
+      name: 'qri.cloud'
+    }
+  }
+
+  if (meta.keywords) jld.keywords = meta.keywords
+  if (meta.license) jld.license = meta.license.url
+
+  return `<script type="application/ld+json">${JSON.stringify(jld, null, 2)}</script>`
+}
+
+exports.composeJSONLD = composeJSONLD