feat: add YNAP parsing to transaction import

This adds YNAP parsing to transaction importing, removing the need to parse the file in the YNAP web app before uploading it to Envelope Zero. Co-Authored-By: Fynn Heintz <[email protected]> Co-Authored-By: Morre <[email protected]>
envelope-zero · Aug 14, 2023 · b1b4c2b · b1b4c2b
1 parent f861350
commit b1b4c2b
Show file tree

Hide file tree

Showing 10 changed files with 78 additions and 62 deletions.
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -14,6 +14,9 @@ env:
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
+    permissions:
+      packages: read
+
     steps:
       - uses: actions/[email protected]
 
@@ -23,18 +26,23 @@ jobs:
           node-version: '16'
           cache: 'npm'
 
+      - name: set npm auth
+        run: echo "//npm.pkg.github.com/:_authToken=${{ secrets.GITHUB_TOKEN }}" > ~/.npmrc
+
       - name: install node modules
         run: npm ci
 
       - uses: pre-commit/[email protected]
 
   cypress:
     runs-on: ubuntu-latest
+    permissions:
+      packages: read
 
     # Current backend version as service container
     services:
       backend:
-        image: ghcr.io/envelope-zero/backend:v3.0.1
+        image: ghcr.io/envelope-zero/backend:v2.10.1
         env:
           CORS_ALLOW_ORIGINS: http://localhost:3001
           API_URL: http://localhost:3001/api
@@ -44,6 +52,9 @@ jobs:
     steps:
       - uses: actions/[email protected]
 
+      - name: set npm auth
+        run: echo "//npm.pkg.github.com/:_authToken=${{ secrets.GITHUB_TOKEN }}" > ~/.npmrc
+
       - uses: cypress-io/[email protected]
         env:
           PORT: 3001
@@ -106,9 +117,9 @@ jobs:
       - name: Build and push Docker image
         uses: docker/[email protected]
         with:
-          context: .
           push: ${{ steps.push_check.outputs.push }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             REACT_APP_VERSION=${{ github.ref_name }}
+            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
diff --git a/.npmrc b/.npmrc
@@ -0,0 +1 @@
+@envelope-zero:registry=https://npm.pkg.github.com
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -4,11 +4,12 @@ Contributions are welcome. Please note the [Code of Conduct](CODE_OF_CONDUCT.md)
 
 ## Tool & Repository setup
 
-You will need the following tools:
+You will need to:
 
-- [pre-commit](https://pre-commit.com/)
+- Install [pre-commit](https://pre-commit.com/)
+- Set up a GitHub Personal Access Token to read from the GitHub package repository. Follow the steps in https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-npm-registry#authenticating-with-a-personal-access-token.
 
-Once those are installed, run `make setup` to perform the repository setup.
+Once these steps are done, run `make setup` to perform the repository setup.
 
 ## Development server
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,10 +1,13 @@
-FROM node:20.5.1-alpine AS builder
+FROM node:20.3.1-alpine AS builder
 ENV NODE_ENV production
 WORKDIR /app
 
 # copy package.json first to avoid unnecessary npm install
 COPY package.json package-lock.json /app/
-RUN npm install --production
+
+ARG GITHUB_TOKEN
+RUN echo "//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}" > ~/.npmrc && \
+    npm install --production
 
 # Copy app files
 COPY src /app/src

diff --git a/cypress/e2e/transaction-import.cy.ts b/cypress/e2e/transaction-import.cy.ts
@@ -35,9 +35,7 @@ describe('Transaction Import', () => {
     cy.contains('My Other Account')
     cy.contains('My Account').click()
 
-    cy.getInputFor('File').selectFile(
-      'cypress/fixtures/transactions-parsed.csv'
-    )
+    cy.getInputFor('File').selectFile('cypress/fixtures/comdirect.csv')
 
     cy.clickAndWait('Submit')
 
@@ -163,9 +161,7 @@ describe('Transaction Import', () => {
     cy.awaitLoading()
 
     cy.getInputFor('Account').type('My account{enter}')
-    cy.getInputFor('File').selectFile(
-      'cypress/fixtures/transactions-parsed.csv'
-    )
+    cy.getInputFor('File').selectFile('cypress/fixtures/comdirect.csv')
 
     cy.clickAndWait('Submit')
     cy.contains('This is a duplicate of an existing transaction')

diff --git a/cypress/fixtures/comdirect.csv b/cypress/fixtures/comdirect.csv
@@ -0,0 +1,11 @@
+;
+"Umsätze Verrechnungskonto ";"Zeitraum: 30 Tage";
+"Neuer Kontostand";"16,94 EUR";
+
+"Buchungstag";"Wertstellung (Valuta)";"Vorgang";"Buchungstext";"Umsatz in EUR";
+"20.06.2023";"20.06.2023";"Irrevevant";"Empfänger: Non Existing Account Buchungstext: Text Ref. 000000000/0 ";"-84,76";
+"19.06.2023";"19.06.2023";"Irrevevant";"Auftraggeber: My Other Account Buchungstext: Transfer from my other account Ref. 0000000000000/0 ";"5,00";
+"16.06.2023";"16.06.2023";"Irrevevant";"Empfänger: New Account Buchungstext: REDACTED Ref. 000000000/0 ";"-784,94";
+"15.06.2023";"15.06.2023";"Irrevevant";"Auftraggeber: External Account Buchungstext: Text Ref. 000000000/0 ";"-52,03";
+
+"Alter Kontostand";"16,89 EUR";
diff --git a/cypress/fixtures/transactions-parsed.csv b/cypress/fixtures/transactions-parsed.csv
diff --git a/package.json b/package.json
@@ -1,6 +1,7 @@
 {
   "name": "envelope-zero-frontend",
   "dependencies": {
+    "@envelope-zero/ynap-parsers": "1.15.11",
     "@headlessui/react": "1.7.16",
     "@heroicons/react": "2.0.18",
     "@tailwindcss/forms": "0.5.4",
@@ -11,6 +12,7 @@
     "@types/node": "18.17.5",
     "@types/react": "18.2.20",
     "@types/react-dom": "18.2.7",
+    "buffer": "6.0.3",
     "flowbite": "1.8.1",
     "flowbite-react": "0.5.0",
     "http-proxy-middleware": "2.0.6",
@@ -50,10 +52,7 @@
     "start": "react-scripts start",
     "build": "react-scripts build",
     "server:dev": "docker-compose --project-name frontend-dev --file docker-compose-dev.yml up",
-    "server:test": "docker-compose --project-name frontend-test --file docker-compose-test.yml up --detach",
-    "server:test:stop": "docker-compose --project-name frontend-test --file docker-compose-test.yml down",
-    "server:test:restart": "docker-compose --project-name frontend-test --file docker-compose-test.yml restart",
-    "server:test:logs": "docker-compose --project-name frontend-test --file docker-compose-test.yml logs --follow",
+    "server:test": "docker-compose --project-name frontend-test --file docker-compose-test.yml up",
     "test": "cypress run",
     "test:watch": "cypress open",
     "eject": "react-scripts eject",

diff --git a/src/components/TransactionImport/Form.tsx b/src/components/TransactionImport/Form.tsx
@@ -2,6 +2,7 @@ import { useTranslation } from 'react-i18next'
 import { useState } from 'react'
 import { Link } from 'react-router-dom'
 import { Account, TransactionPreview, Translation } from '../../types'
+import { parseFile } from '@envelope-zero/ynap-parsers'
 import { checkStatus, parseJSON } from '../../lib/fetch-helper'
 import Error from '../Error'
 import FormFields from '../FormFields'
@@ -28,25 +29,38 @@ const Form = ({ accounts, isLoading, setIsLoading, setResult }: Props) => {
       onSubmit={event => {
         event.preventDefault()
 
-        setIsLoading(true)
+        const file = (event.target as any).file.files[0] // TODO: without `any`
 
-        fetch(`/api/v1/import/ynab-import-preview?accountId=${accountId}`, {
-          method: 'POST',
-          body: new FormData(event.target as HTMLFormElement),
-        })
-          .then(checkStatus)
-          .then(parseJSON)
-          .then(body => {
-            setIsLoading(false)
-            if (error) {
-              setError('')
-            }
-            setResult(body.data, accountId)
-          })
-          .catch(error => {
-            setIsLoading(false)
-            setError(error.message)
+        parseFile(file)
+          .then(result => {
+            // Get the data and transform it to FormData for the backend
+            const data = new FormData()
+            data.append(
+              'file',
+              new File([new Blob([result[0].data])], 'ynap-transactions.csv', {
+                type: 'text/csv;charset=utf-8;',
+              })
+            )
+
+            fetch(`/api/v1/import/ynab-import-preview?accountId=${accountId}`, {
+              method: 'POST',
+              body: data,
+            })
+              .then(checkStatus)
+              .then(parseJSON)
+              .then(body => {
+                setIsLoading(false)
+                if (error) {
+                  setError('')
+                }
+                setResult(body.data, accountId)
+              })
+              .catch(error => {
+                setIsLoading(false)
+                setError(error.message)
+              })
           })
+          .catch(error => setError(error))
       }}
     >
       <div className="header">
@@ -64,20 +78,12 @@ const Form = ({ accounts, isLoading, setIsLoading, setResult }: Props) => {
       ) : (
         <>
           <Error error={error} />
-          <p className="pt-4 whitespace-pre-line dark:text-gray-400">
-            {t('transactions.import.description')}
-          </p>
-          <h2 className="mt-4">{t('transactions.import.howTo.title')}</h2>
-          <ol className="list-decimal ml-4 pt-4 whitespace-pre-line dark:text-gray-400">
-            {['download', 'parse', 'upload'].map(step => (
-              <li
-                key={step}
-                dangerouslySetInnerHTML={{
-                  __html: t(`transactions.import.howTo.${step}`),
-                }}
-              />
-            ))}
-          </ol>
+          <p
+            className="pt-4 whitespace-pre-line dark:text-gray-400"
+            dangerouslySetInnerHTML={{
+              __html: t('transactions.import.description'),
+            }}
+          ></p>
           <FormFields>
             <Autocomplete<Account>
               groups={[{ items: accounts }]}

diff --git a/src/translations/en.json b/src/translations/en.json
@@ -88,20 +88,13 @@
     "import": {
       "importTransactions": "Import Transactions",
       "account": "Account",
-      "description": "Import a list of transactions (for example a downloaded bank statement). \n\nYou will be able to review the parsed transactions before they are saved.",
-      "ynapCredits": "Uploads are parsed in your browser, powered by <a class='link-blue' href='https://ynap.leolabs.org/'>YNAP</a>. A list of all currently supported banks can be found <a class='link-blue' href='https://ynap.leolabs.org/supported-formats/'>here</a>.",
+      "description": "Import a list of transactions, for example a downloaded bank statement.\n\nYou will be able to review the transactions before they are saved.\n\nUploads are parsed in your browser, powered by <a class='link-blue' href='https://l.envelope-zero.org/ynap'>YNAP</a>, and by Envelope Zero. A list of all currently supported banks can be found <a class='link-blue' href='https://l.envelope-zero.org/ynap-formats'>here</a>.",
       "previous": "Previous Transaction",
       "next": "Next Transaction",
       "importSuccess": "Successfully imported",
       "deleteSuccess": "Successfully deleted",
       "duplicateDetected": "This is a duplicate of an existing transaction",
-      "complete": "Import complete",
-      "howTo": {
-        "title": "How To Import",
-        "download": "Download a .CSV export of your transactions from your bank",
-        "parse": "Parse that file using <a class='link-blue' href='https://ynap.leolabs.org/'>YNAP</a> (A list of all currently supported banks can be found <a class='link-blue' href='https://ynap.leolabs.org/supported-formats/'>here</a>)",
-        "upload": "Upload the parsed file here:"
-      }
+      "complete": "Import complete"
     },
     "from": "From",
     "to": "To",
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		@envelope-zero:registry=https://npm.pkg.github.com