merge upstream

MANIFEST.in

@@ -3,11 +3,14 @@ include README.rst
 include electrum-bitg.desktop
 include *.py
 include run_electrum
-include contrib/requirements/requirements.txt
-include contrib/requirements/requirements-hw.txt
 recursive-include packages *.py
 recursive-include packages cacert.pem
 
+include contrib/requirements/requirements*.txt
+include contrib/deterministic-build/requirements*.txt
+include contrib/make_libsecp256k1.sh
+include contrib/build_tools_util.sh
+
 graft electrum
 prune electrum/tests
 graft contrib/udev

RELEASE-NOTES

@@ -1,3 +1,22 @@
+# Release 4.0.3 - (Sep 11, 2020)
+ * PSBT: restore compatibility with Bitcoin Core following CVE-2020-14199:
+   we now allow a PSBT input to have both UTXO and WITNESS_UTXO (#6429).
+   (PSBTs created since 4.0.1 already contained UTXO for segwit inputs)
+ * Hardware wallets:
+   - bitbox02: better multisig UX: implement get_soft_device_id (#6386)
+   - coldcard: fix "show address" for multisig (#6517)
+   - all: run all device communication on a dedicated thread (#6561).
+     This should resolve some threading issues.
+ * new feature: "Automated BIP39 recovery" (#6219, #6155)
+   When restoring from a BIP39 seed, add option to scan many known
+   derivation paths for history, and show them to user to choose from.
+ * show derivation path of keystores in Qt GUI Wallet>Information (#4700)
+ * fix "signtransaction" RPC command (#6502)
+ * Dependencies: pyaes is no longer needed (#6563)
+ * The tar.gz source dist now bundles make_libsecp256k1.sh, to help
+   users getting libsecp256k1 (#6323).
+ * A few other minor bugfixes and usability improvements.
+
 # Release 4.0.2 - (July 8, 2020)
  - rm old corrupted non-bip70 invoices (#6345)
  - other minor fixes

contrib/android/buildozer.spec

@@ -38,8 +38,8 @@ version.filename = %(source.dir)s/electrum/version.py
 # (list) Application requirements
 requirements =
     # note: re python3.8, see #6147
-    hostpython3==3.7.7,
-    python3==3.7.7,
+    hostpython3==3.7.9,
+    python3==3.7.9,
     android,
     openssl,
     plyer,

contrib/build-linux/appimage/Dockerfile

@@ -11,15 +11,15 @@ RUN apt-get update -q && \
         autoconf=2.69-9 \
         libtool=2.4.6-0.1 \
         xz-utils=5.1.1alpha+20120614-2ubuntu2 \
-        libssl-dev=1.0.2g-1ubuntu4.16 \
-        libssl1.0.0=1.0.2g-1ubuntu4.16 \
-        openssl=1.0.2g-1ubuntu4.16 \
+        libssl-dev=1.0.2g-1ubuntu4.17 \
+        libssl1.0.0=1.0.2g-1ubuntu4.17 \
+        openssl=1.0.2g-1ubuntu4.17 \
         zlib1g-dev=1:1.2.8.dfsg-2ubuntu4.3 \
         libffi-dev=3.2.1-4 \
         libncurses5-dev=6.0+20160213-1ubuntu1 \
         libsqlite3-dev=3.11.0-1ubuntu1.5 \
         libusb-1.0-0-dev=2:1.0.20-1 \
-        libudev-dev=229-4ubuntu21.28 \
+        libudev-dev=229-4ubuntu21.29 \
         gettext=0.19.7-2ubuntu3.1 \
         libzbar0=0.10+doc-10ubuntu1  \
         libdbus-1-3=1.10.6-1ubuntu3.6 \

contrib/build-linux/appimage/build.sh

@@ -13,7 +13,7 @@ CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage"
 export GCC_STRIP_BINARIES="1"
 
 # pinned versions
-PYTHON_VERSION=3.7.7
+PYTHON_VERSION=3.7.9
 PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15"
 SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386"
 
@@ -38,7 +38,7 @@ download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/AppImage/AppI
 verify_hash "$CACHEDIR/appimagetool" "d918b4df547b388ef253f3c9e7f6529ca81a885395c31f619d9aaf7030499a13"
 
 download_if_not_exist "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz"
-verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "06a0a9f1bf0d8cd1e4121194d666c4e28ddae4dd54346de6c343206599f02136"
+verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "91923007b05005b5f9bd46f3b9172248aea5abc1543e8a636d59e629c3331b01"
 
 
 
@@ -113,8 +113,11 @@ info "preparing electrum-locale."
 )
 
 
-info "installing electrum-bitg and its dependencies."
+info "Installing build dependencies."
 mkdir -p "$CACHEDIR/pip_cache"
+"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-appimage-build.txt"
+
+info "installing electrum and its dependencies."
 "$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
 "$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
 "$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"

contrib/build-wine/Dockerfile

@@ -6,8 +6,8 @@ RUN dpkg --add-architecture i386 && \
     apt-get update -q && \
     apt-get install -qy \
         wget=1.19.4-1ubuntu2.2 \
-        gnupg2=2.2.4-1ubuntu1.2 \
-        dirmngr=2.2.4-1ubuntu1.2 \
+        gnupg2=2.2.4-1ubuntu1.3 \
+        dirmngr=2.2.4-1ubuntu1.3 \
         python3-software-properties=0.96.24.32.1 \
         software-properties-common=0.96.24.32.1
 
@@ -17,8 +17,11 @@ RUN apt-get update -q && \
         p7zip-full=16.02+dfsg-6 \
         make=4.1-9.1ubuntu1 \
         mingw-w64=5.0.3-1 \
+        mingw-w64-tools=5.0.3-1 \
+        win-iconv-mingw-w64-dev=0.0.8-2 \
         autotools-dev=20180224.1 \
         autoconf=2.69-11 \
+        autopoint=0.19.8.1-6ubuntu0.3 \
         libtool=2.4.6-2 \
         gettext=0.19.8.1-6
 

contrib/build-wine/build.sh

@@ -11,7 +11,7 @@ export CACHEDIR="$here/.cache"
 export PIP_CACHE_DIR="$CACHEDIR/pip_cache"
 
 export BUILD_TYPE="wine"
-export GCC_TRIPLET_HOST="i686-w64-mingw32"
+export GCC_TRIPLET_HOST="i686-w64-mingw32"  # make sure to clear caches if changing this
 export GCC_TRIPLET_BUILD="x86_64-pc-linux-gnu"
 export GCC_STRIP_BINARIES="1"
 
@@ -29,6 +29,12 @@ else
     "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
 fi
 
+if [ -f "$PROJECT_ROOT/electrum/libzbar-0.dll" ]; then
+    info "libzbar already built, skipping"
+else
+    "$CONTRIB"/make_zbar.sh || fail "Could not build zbar"
+fi
+
 $here/prepare-wine.sh || fail "prepare-wine failed"
 
 info "Resetting modification time in C:\Python..."

contrib/build-wine/deterministic.spec

@@ -34,14 +34,14 @@ binaries += [b for b in collect_dynamic_libs('PyQt5') if 'qwindowsvista' in b[0]
 
 binaries += [('C:/tmp/libsecp256k1-0.dll', '.')]
 binaries += [('C:/tmp/libusb-1.0.dll', '.')]
+binaries += [('C:/tmp/libzbar-0.dll', '.')]
 
 datas = [
     (home+'electrum/*.json', 'electrum'),
     (home+'electrum/lnwire/*.csv', 'electrum/lnwire'),
     (home+'electrum/wordlist/english.txt', 'electrum/wordlist'),
     (home+'electrum/locale', 'electrum/locale'),
     (home+'electrum/plugins', 'electrum/plugins'),
-    ('C:\\Program Files (x86)\\ZBar\\bin\\', '.'),
     (home+'electrum/gui/icons', 'electrum/gui/icons'),
 ]
 datas += collect_data_files('trezorlib')

contrib/build-wine/prepare-wine.sh

@@ -2,22 +2,18 @@
 
 # Please update these carefully, some versions won't work under Wine
 NSIS_FILENAME=nsis-3.05-setup.exe
-NSIS_URL=https://prdownloads.sourceforge.net/nsis/$NSIS_FILENAME?download
+NSIS_URL=https://downloads.sourceforge.net/project/nsis/NSIS%203/3.05/$NSIS_FILENAME
 NSIS_SHA256=1a3cc9401667547b9b9327a177b13485f7c59c2303d4b6183e7bc9e6c8d6bfdb
 
-ZBAR_FILENAME=zbarw-20121031-setup.exe
-ZBAR_URL=https://sourceforge.net/projects/zbarw/files/$ZBAR_FILENAME/download
-ZBAR_SHA256=177e32b272fa76528a3af486b74e9cb356707be1c5ace4ed3fcee9723e2c2c02
-
 LIBUSB_REPO="https://github.com/libusb/libusb.git"
 LIBUSB_COMMIT="e782eeb2514266f6738e242cdcb18e3ae1ed06fa"
 # ^ tag v1.0.23
 
 PYINSTALLER_REPO="https://github.com/SomberNight/pyinstaller.git"
-PYINSTALLER_COMMIT="e934539374e30d1500fcdbe8e4eb0860413935b2"
-# ^ tag 3.6, plus a custom commit that fixes cross-compilation with MinGW
+PYINSTALLER_COMMIT="6e455b2c1208465742484436009bfb1e1baf2e01"
+# ^ tag 4.0, plus a custom commit that fixes cross-compilation with MinGW
 
-PYTHON_VERSION=3.7.7
+PYTHON_VERSION=3.7.9
 
 ## These settings probably don't need change
 export WINEPREFIX=/opt/wine64
@@ -47,12 +43,19 @@ info "Installing Python."
 # keys from https://www.python.org/downloads/#pubkeys
 KEYRING_PYTHON_DEV="keyring-electrum-build-python-dev.gpg"
 gpg --no-default-keyring --keyring $KEYRING_PYTHON_DEV --import "$here"/gpg_keys/7ED10B6531D7C8E1BC296021FC624643487034E5.asc
-PYTHON_DOWNLOADS="$CACHEDIR/python$PYTHON_VERSION"
+if [ "$GCC_TRIPLET_HOST" = "i686-w64-mingw32" ] ; then
+    ARCH="win32"
+elif [ "$GCC_TRIPLET_HOST" = "x86_64-w64-mingw32" ] ; then
+    ARCH="amd64"
+else
+    fail "unexpected GCC_TRIPLET_HOST: $GCC_TRIPLET_HOST"
+fi
+PYTHON_DOWNLOADS="$CACHEDIR/python$PYTHON_VERSION-$ARCH"
 mkdir -p "$PYTHON_DOWNLOADS"
 for msifile in core dev exe lib pip tools; do
     echo "Installing $msifile..."
-    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi" "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi"
-    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi.asc" "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi.asc"
+    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi" "https://www.python.org/ftp/python/$PYTHON_VERSION/$ARCH/${msifile}.msi"
+    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi.asc" "https://www.python.org/ftp/python/$PYTHON_VERSION/$ARCH/${msifile}.msi.asc"
     verify_signature "$PYTHON_DOWNLOADS/${msifile}.msi.asc" $KEYRING_PYTHON_DEV
     wine msiexec /i "$PYTHON_DOWNLOADS/${msifile}.msi" /qb TARGETDIR=$PYHOME
 done
@@ -63,11 +66,6 @@ $PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"
 info "Installing dependencies specific to binaries."
 $PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
 
-info "Installing ZBar."
-download_if_not_exist "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_URL"
-verify_hash "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_SHA256"
-wine "$CACHEDIR/$ZBAR_FILENAME" /S
-
 info "Installing NSIS."
 download_if_not_exist "$CACHEDIR/$NSIS_FILENAME" "$NSIS_URL"
 verify_hash "$CACHEDIR/$NSIS_FILENAME" "$NSIS_SHA256"
@@ -91,18 +89,19 @@ info "Compiling libusb..."
     git checkout -b pinned "${LIBUSB_COMMIT}^{commit}"
     echo "libusb_1_0_la_LDFLAGS += -Wc,-static" >> libusb/Makefile.am
     ./bootstrap.sh || fail "Could not bootstrap libusb"
-    host="i686-w64-mingw32"
+    host="$GCC_TRIPLET_HOST"
     LDFLAGS="-Wl,--no-insert-timestamp" ./configure \
         --host=$host \
-        --build=x86_64-pc-linux-gnu || fail "Could not run ./configure for libusb"
+        --build=$GCC_TRIPLET_BUILD || fail "Could not run ./configure for libusb"
     make -j4 || fail "Could not build libusb"
     ${host}-strip libusb/.libs/libusb-1.0.dll
 ) || fail "libusb build failed"
 cp "$CACHEDIR/libusb/libusb/.libs/libusb-1.0.dll" $WINEPREFIX/drive_c/tmp/  || fail "Could not copy libusb to its destination"
 
 
-# copy libsecp dll (already built)
+# copy already built DLLs
 cp "$PROJECT_ROOT/electrum/libsecp256k1-0.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libsecp to its destination"
+cp "$PROJECT_ROOT/electrum/libzbar-0.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libzbar to its destination"
 
 
 info "Building PyInstaller."
@@ -126,10 +125,21 @@ info "Building PyInstaller."
     echo "const char *electrum_tag = \"tagged by Electrum-BITG@$ELECTRUM_COMMIT_HASH\";" >> ./bootloader/src/pyi_main.c
     pushd bootloader
     # cross-compile to Windows using host python
-    python3 ./waf all CC=i686-w64-mingw32-gcc CFLAGS="-static -Wno-dangling-else -Wno-error=unused-value"
+    python3 ./waf all CC="${GCC_TRIPLET_HOST}-gcc" \
+                      CFLAGS="-static \
+                              -Wno-dangling-else \
+                              -Wno-error=unused-value \
+                              -Wno-error=implicit-function-declaration \
+                              -Wno-error=int-to-pointer-cast"
     popd
     # sanity check bootloader is there:
-    [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir!"
+    if [ "$GCC_TRIPLET_HOST" = "i686-w64-mingw32" ] ; then
+        [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir! (32bit)"
+    elif [ "$GCC_TRIPLET_HOST" = "x86_64-w64-mingw32" ] ; then
+        [[ -e PyInstaller/bootloader/Windows-64bit/runw.exe ]] || fail "Could not find runw.exe in target dir! (64bit)"
+    else
+        fail "unexpected GCC_TRIPLET_HOST: $GCC_TRIPLET_HOST"
+    fi
 ) || fail "PyInstaller build failed"
 info "Installing PyInstaller."
 $PYTHON -m pip install --no-dependencies --no-warn-script-location ./pyinstaller

contrib/deterministic-build/requirements-appimage-build.txt

@@ -0,0 +1,43 @@
+Cython==0.29.21 \
+    --hash=sha256:0ac10bf476476a9f7ef61ec6e44c280ef434473124ad31d3132b720f7b0e8d2a \
+    --hash=sha256:0e25c209c75df8785480dcef85db3d36c165dbc0f4c503168e8763eb735704f2 \
+    --hash=sha256:171b9f70ceafcec5852089d0f9c1e75b0d554f46c882cd4e2e4acaba9bd7d148 \
+    --hash=sha256:23f3a00b843a19de8bb4468b087db5b413a903213f67188729782488d67040e0 \
+    --hash=sha256:2922e3031ba9ebbe7cb9200b585cc33b71d66023d78450dcb883f824f4969371 \
+    --hash=sha256:31c71a615f38401b0dc1f2a5a9a6c421ffd8908c4cd5bbedc4014c1b876488e8 \
+    --hash=sha256:473df5d5e400444a36ed81c6596f56a5b52a3481312d0a48d68b777790f730ae \
+    --hash=sha256:497841897942f734b0abc2dead2d4009795ee992267a70a23485fd0e937edc0b \
+    --hash=sha256:539e59949aab4955c143a468810123bf22d3e8556421e1ce2531ed4893914ca0 \
+    --hash=sha256:540b3bee0711aac2e99bda4fa0a46dbcd8c74941666bfc1ef9236b1a64eeffd9 \
+    --hash=sha256:57ead89128dee9609119c93d3926c7a2add451453063147900408a50144598c6 \
+    --hash=sha256:5c4276fdcbccdf1e3c1756c7aeb8395e9a36874fa4d30860e7694f43d325ae13 \
+    --hash=sha256:5da187bebe38030325e1c0b5b8a804d489410be2d384c0ef3ba39493c67eb51e \
+    --hash=sha256:5e545a48f919e40079b0efe7b0e081c74b96f9ef25b9c1ff4cdbd95764426b58 \
+    --hash=sha256:603b9f1b8e93e8b494d3e89320c410679e21018e48b6cbc77280f5db71f17dc0 \
+    --hash=sha256:695a6bcaf9e12b1e471dfce96bbecf22a1487adc2ac6106b15960a2b51b97f5d \
+    --hash=sha256:715294cd2246b39a8edca464a8366eb635f17213e4a6b9e74e52d8b877a8cb63 \
+    --hash=sha256:856c7fb31d247ce713d60116375e1f8153d0291ab5e92cca7d8833a524ba9991 \
+    --hash=sha256:8c6e25e9cc4961bb2abb1777c6fa9d0fa2d9b014beb3276cebe69996ff162b78 \
+    --hash=sha256:9207fdedc7e789a3dcaca628176b80c82fbed9ae0997210738cbb12536a56699 \
+    --hash=sha256:93f5fed1c9445fb7afe20450cdaf94b0e0356d47cc75008105be89c6a2e417b1 \
+    --hash=sha256:9ce5e5209f8406ffc2b058b1293cce7a954911bb7991e623564d489197c9ba30 \
+    --hash=sha256:a0674f246ad5e1571ef29d4c5ec1d6ecabe9e6c424ad0d6fee46b914d5d24d69 \
+    --hash=sha256:b2f9172e4d6358f33ecce6a4339b5960f9f83eab67ea244baa812737793826b7 \
+    --hash=sha256:b8a8a31b9e8860634adbca30fea1d0c7f08e208b3d7611f3e580e5f20992e5d7 \
+    --hash=sha256:b8d8497091c1dc8705d1575c71e908a93b1f127a174b2d472020f3d84263ac28 \
+    --hash=sha256:c4b78356074fcaac04ecb4de289f11d506e438859877670992ece11f9c90f37b \
+    --hash=sha256:c541b2b49c6638f2b5beb9316726db84a8d1c132bf31b942dae1f9c7f6ad3b92 \
+    --hash=sha256:c8435959321cf8aec867bbad54b83b7fb8343204b530d85d9ea7a1f5329d5ac2 \
+    --hash=sha256:ccb77faeaad99e99c6c444d04862c6cf604204fe0a07d4c8f9cbf2c9012d7d5a \
+    --hash=sha256:e272ed97d20b026f4f25a012b25d7d7672a60e4f72b9ca385239d693cd91b2d5 \
+    --hash=sha256:e57acb89bd55943c8d8bf813763d20b9099cc7165c0f16b707631a7654be9cad \
+    --hash=sha256:e93acd1f603a0c1786e0841f066ae7cef014cf4750e3cd06fd03cfdf46361419
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e