Merge branch 'main' into download-data

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
           pipenv install --dev
 
       - name: Run tests
-        run: pipenv run pytest
+        run: pipenv run test
 
   ruff:
     runs-on: ubuntu-latest

.gitignore

@@ -1,5 +1,13 @@
-data
-out
+.coverage
 .env
 .pytest_cache
-__pycache__
+__pycache__/
+data/
+interaction-matrix-of-user
+movie_titles.txt
+mv_0000001.txt
+mv_0000002.txt
+myenv/
+nf_prize_dataset.tar.gz
+out/
+output/

.vscode/extensions.json

@@ -1,6 +1,7 @@
 {
 	"recommendations": [
-		"charliermarsh.ruff"
+		"charliermarsh.ruff",
+		"mechatroner.rainbow-csv"
 	],
 	"unwantedRecommendations": [
 

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Debug: Main Module",
+            "args": ["-v"],
+            "type": "debugpy",
+            "request": "launch",
+            "module": "mediabridge.main"
+        }
+    ]
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Travis Briggs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Pipfile

@@ -4,18 +4,24 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
+datasette = "~=0.65"
+numpy = "~=1.26"
+pandas = "~=2.2"
 pymongo = "~=4.10"
 python-dotenv = "~=1.0"
-requests = "~=2.32"
-ruff = "~=0.7"
-tqdm = "~=4.66"
-wget = "~=3.2"
 rectools-lightfm = "~=1.17.3"
+requests = "~=2.26"
+ruff = "~=0.7"
+scikit-surprise = "~=1.1"
 scipy = "~=1.14"
+sqlalchemy = "~=2.0"
+tqdm = "~=4.66"
 typer = "~=0.12"
+wget = "~=3.2"
 
 [dev-packages]
 mypy = "~=1.14"
+pandas-stubs = "~=2.2"
 pyright = "~=1.1"
 pytest = "~=8.3"
 pytest-cov = "~=5.0"
@@ -27,9 +33,10 @@ types-tqdm = "~=4.67"
 python_version = "3.12"
 
 [scripts]
-download_data = "pipenv run python -m mediabridge.data_processing.download_data"
-delete_data = "pipenv run python -m mediabridge.data_processing.clean_up_data"
+browse = "pipenv run datasette serve out/movies.sqlite"
 coverage = "pipenv run bin/coverage.sh"
+delete_data = "pipenv run python -m mediabridge.data_processing.clean_up_data"
 dev = "pipenv run python -m mediabridge.main"
+download_data = "pipenv run python -m mediabridge.data_processing.download_data"
 lint = "pipenv run bin/lint.sh"
-test = "pipenv run python -m pytest"
+test = "pipenv run python -m pytest"

README.md

@@ -1,6 +1,6 @@
 # What is MediaBridge?
 
-MediaBridge is a project being developed at the [Noisebridge](https://github.com/noisebridge) hackerspace in San Francisco, CA, USA. See also the [Noisebridge hompage](https://www.noisebridge.net/wiki/Noisebridge) and the [wiki entry for this project](https://www.noisebridge.net/wiki/Python_Project_Meetup).
+MediaBridge is a project being developed at the [Noisebridge](https://github.com/noisebridge) hackerspace in San Francisco, CA, USA. See also the [Noisebridge homepage](https://www.noisebridge.net/wiki/Noisebridge) and the [wiki entry for this project](https://www.noisebridge.net/wiki/Python_Project_Meetup).
 
 MediaBridge is in a _very_ early stage of the development. It's intended functionality is to provide recommendations that _bridge_ media types. So for example, you might say you're interested in the film _Saw_ and MediaBrige might recommend the video game _Silent Hill_ or a Stephen King book. For now, we are working on simply returning recommendations for movies, based on the [Netflix Prize dataset](https://www.kaggle.com/datasets/netflix-inc/netflix-prize-data).
 
@@ -12,34 +12,39 @@ This code uses Python 3. It is tested on Python 3.12, but will probably work on
 
 To install the project dependencies, first install pipenv globally with `pip install pipenv`. Then create a virtual env/install dependencies with `pipenv install --dev`.
 
-To run code in the project, prefix your command with `pipenv run`, a la `pipenv run python -m mediabridge.main`.
+To run code in the pipenv virtual environment, prefix your command with `pipenv run` (ex. `pipenv run python` runs the python interpreter in the pipenv environment).
 
-## Running main
+### Using the pipenv environment in VSCode
 
-The "main.py" script is part of the `mediabridge` module. Additionally, running it requires `pipenv run` as mentioned above. So the full command to run the main script (or any other script in the `mediabridge` module) is:
+To fix import errors and other Intellisense features, make sure you've let VSCode know about your pipenv environment. To do that:
+
+1. Open the VSCode command palette (Control/Command+SHIFT+P)
+2. Search for and select the "Python: Select Interpreter" command
+3. Choose the option that starts with `MediaBridge`
+
+## Running code
+
+For development purposes, you can simply run the dev script:
 
 ```
-pipenv run python -m mediabridge.main
+pipenv run dev
 ```
 
-This should be run from the root of the project directory.
+Be sure to specify options such as -v and -l *before* any subcommands (process, load, etc.).
 
-### Running from VSCode
+**NOTE:** *If you encounter a ModuleNotFoundError, make sure you are in the root directory of the project, as the `mediabridge` directory is the module Pipenv is trying to reference.*
 
-To fix import errors and other Intellisense features, make sure you've let VSCode know about your pipenv environment. To do that:
-
-1. Open the VSCode command palette (Control/Command+SHIFT+P)
-2. Search for and select the "Python: Select Interpreter" command
-3. Choose the option that starts with `MediaBridge`
+This is currently just an alias to run the main script using `pipenv run python -m mediabridge.main`, but this may change in the future, so using `pipenv run dev` will ensure the correct script is always run.
 
 ## Testing
 
-To run unit tests,
+To run unittests:
 
-1. Ensure `pipenv` is installed
-2. Run `pipenv run pytest`
+```
+pipenv run test
+```
 
-There is a GitHub actions "check" for passing tests, which must pass for you to be able to merge your PR.
+These tests are also evaluated via a GitHub action when opening or updating a PR and must pass before merging.
 
 ## Code formatting
 

bin/coverage.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+# usage: bin/coverage.sh
+
+set -e
+source bin/util.sh
+
+ENV="env PYTHONPATH=.:../..:${PYTHONPATH}"
+
+coverage erase
+${ENV} coverage run -p -m pytest -v
+# It's unclear where files like mv_0000002.txt come from. Unable to reproduce.
+# ${ENV} coverage run -p mediabridge/data_processing/interaction_matrix.py
+coverage run -p mediabridge/definitions.py 2> /dev/null || true
+coverage run -p -m mediabridge.definitions
+coverage run -p -m unittest mediabridge/*/*_test.py
+coverage combine --quiet
+coverage html
+coverage report
+
+cat <<EOF
+View the coverage report in a web browser:
+open htmlcov/index.html
+EOF

bin/lint.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+# usage: bin/lint.sh
+
+set -e
+source bin/util.sh
+
+${ENV} ruff check .
+
+${ENV} mypy --strict --show-traceback --warn-unreachable --ignore-missing-imports --no-namespace-packages .
+
+${ENV} pyright .

bin/test.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+
+# usage: bin/test.sh
+
+set -e
+source bin/util.sh
+
+${ENV} python -m unittest mediabridge/*/*_test.py
+
+${ENV} pytest

bin/util.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+set -e
+cd ../MediaBridge/mediabridge  # run scripts from top-level of the repo
+# Alas, MacOS is case-insensitive, so this little dance ensures we're at repo top.
+cd ..
+cd ../MediaBridge
+
+if [ -d .venv ]
+then
+    source .venv/bin/activate
+fi
+
+export ENV="env PYTHONPATH=.:../.."
+
+set -x

mediabridge-frontend/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

mediabridge-frontend/README.md

@@ -0,0 +1,50 @@
+# React + TypeScript + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
+
+- Configure the top-level `parserOptions` property like this:
+
+```js
+export default tseslint.config({
+  languageOptions: {
+    // other options...
+    parserOptions: {
+      project: ['./tsconfig.node.json', './tsconfig.app.json'],
+      tsconfigRootDir: import.meta.dirname,
+    },
+  },
+})
+```
+
+- Replace `tseslint.configs.recommended` to `tseslint.configs.recommendedTypeChecked` or `tseslint.configs.strictTypeChecked`
+- Optionally add `...tseslint.configs.stylisticTypeChecked`
+- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and update the config:
+
+```js
+// eslint.config.js
+import react from 'eslint-plugin-react'
+
+export default tseslint.config({
+  // Set the react version
+  settings: { react: { version: '18.3' } },
+  plugins: {
+    // Add the react plugin
+    react,
+  },
+  rules: {
+    // other rules...
+    // Enable its recommended rules
+    ...react.configs.recommended.rules,
+    ...react.configs['jsx-runtime'].rules,
+  },
+})
+```

mediabridge-frontend/components.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "new-york",
+  "rsc": false,
+  "tsx": true,
+  "tailwind": {
+    "config": "tailwind.config.js",
+    "css": "src/index.css",
+    "baseColor": "slate",
+    "cssVariables": true,
+    "prefix": ""
+  },
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils",
+    "ui": "@/components/ui",
+    "lib": "@/lib",
+    "hooks": "@/hooks"
+  },
+  "iconLibrary": "lucide"
+}

mediabridge-frontend/eslint.config.js

@@ -0,0 +1,28 @@
+import js from '@eslint/js'
+import globals from 'globals'
+import reactHooks from 'eslint-plugin-react-hooks'
+import reactRefresh from 'eslint-plugin-react-refresh'
+import tseslint from 'typescript-eslint'
+
+export default tseslint.config(
+  { ignores: ['dist'] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.recommended],
+    files: ['**/*.{ts,tsx}'],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+    },
+    plugins: {
+      'react-hooks': reactHooks,
+      'react-refresh': reactRefresh,
+    },
+    rules: {
+      ...reactHooks.configs.recommended.rules,
+      'react-refresh/only-export-components': [
+        'warn',
+        { allowConstantExport: true },
+      ],
+    },
+  },
+)

mediabridge-frontend/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Mediabridge</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>