aboutcode-org · pombredanne · Oct 11, 2021 · Jun 17, 2021 · Jun 17, 2021 · Aug 31, 2021
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@
 /Lib
 /pip-selfcheck.json
 /tmp
+/venv
 .Python
 /include
 /Include

diff --git a/.travis.yml b/.travis.yml
@@ -19,4 +19,4 @@ python:
 install: ./configure --dev
 
 # Scripts to run at script stage
-script: tmp/bin/pytest --ignore=tests/test_vmimage.py
+script: venv/bin/pytest  --ignore=tests/test_vmimage.py
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,22 @@
 Changelog
 =========
 
+v30.0.0 (next)
+--------
+
+- Update to the latest skeleton. The virtualenv is now created under the venv
+  directory with the ./configure --dev
+- Switch back to semver versioning. Calver was not really needed here.
+- Do not crash if there is a corrupted archive with the --replace-originals
+  option
+
+Thank you to:
+
+- Jono Yang @JonoYang
+- Bryan Sutula @sutula
+- Smascer @Smascer
+- Chin-Yeung Li @chinyeungli
+
 v21.7.23
 --------
 

diff --git a/README.rst b/README.rst
@@ -153,7 +153,8 @@ Development
 
 To set up the development environment::
 
-    source configure --dev
+    configure --dev
+    source venv/bin/activate
 
 
 To run unit tests::
@@ -262,14 +263,14 @@ similar to ExtractCode but with different goals:
 
 
 
-
 List of supported archive formats
 -------------------------------------
 
 ExtractCode can extract the folowing archives formats:
 
 Archive format kind: docs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   name: Office doc
      - extensions: .docx, .dotx, .docm, .xlsx, .xltx, .xlsm, .xltm, .pptx, .ppsx, .potx, .pptm, .potm, .ppsm, .odt, .odf, .sxw, .stw, .ods, .ots, .sxc, .stc, .odp, .otp, .odg, .otg, .sxi, .sti, .sxd, .sxg, .std, .sdc, .sda, .sdd, .smf, .sdw, .sxm, .stw, .oxt, .sldx, .epub
      - filetypes : zip archive, microsoft word 2007+, microsoft excel 2007+, microsoft powerpoint 2007+
@@ -292,6 +293,7 @@ Archive format kind: docs
 
 Archive format kind: regular
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   name: Tar
      - extensions: .tar
      - filetypes : .tar, tar archive
@@ -359,6 +361,7 @@ Archive format kind: regular
 
 Archive format kind: regular_nested
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   name: Tar xz
      - extensions: .tar.xz, .txz, .tarxz
      - filetypes : xz compressed
@@ -414,8 +417,10 @@ Archive format kind: regular_nested
      - filetypes : compress'd data
      - mimetypes : application/x-compress
 
+
 Archive format kind: package
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   name: Ruby Gem package
      - extensions: .gem
      - filetypes : .tar, tar archive
@@ -508,6 +513,7 @@ Archive format kind: package
 
 Archive format kind: file_system
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   name: ISO CD image
      - extensions: .iso, .udf, .img
      - filetypes : iso 9660 cd-rom, high sierra cd-rom
@@ -535,13 +541,15 @@ Archive format kind: file_system
 
 Archive format kind: patches
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   name: Patch
      - extensions: .diff, .patch
      - filetypes : diff, patch
      - mimetypes : text/x-diff
 
 Archive format kind: special_package
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   name: InstallShield Installer
      - extensions: .exe
      - filetypes : installshield

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -21,44 +21,44 @@ jobs:
           image_name: ubuntu-18.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: sudo chmod 0644 /boot/vmlinuz-* && sudo apt install libguestfs-tools && tmp/bin/pytest -n 2 -vvs
+              all: sudo chmod 0644 /boot/vmlinuz-* && sudo apt install libguestfs-tools && venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu20_cpython
           image_name: ubuntu-20.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: sudo chmod 0644 /boot/vmlinuz-* && sudo apt install libguestfs-tools && tmp/bin/pytest -n 2 -vvs
+              all: sudo chmod 0644 /boot/vmlinuz-* && sudo apt install libguestfs-tools && venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1014_cpython
           image_name: macos-10.14
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1015_cpython
           image_name: macos-10.15
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2016_cpython
           image_name: vs2017-win2016
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp\Scripts\pytest -n 2 -vvs
+              all: venv\Scripts\pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2019_cpython
           image_name: windows-2019
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp\Scripts\pytest -n 2 -vvs
+              all: venv\Scripts\pytest -n 2 -vvs
diff --git a/configure b/configure
@@ -26,16 +26,16 @@ CLI_ARGS=$1
 ################################
 
 # Requirement arguments passed to pip and used by default or with --dev.
-REQUIREMENTS="--editable .[full]"
-DEV_REQUIREMENTS="--editable .[full,testing]"
+REQUIREMENTS="--editable .[full] --constraint requirements.txt"
+DEV_REQUIREMENTS="--editable .[full,testing] --constraint requirements.txt --constraint requirements-dev.txt"
 
 # where we create a virtualenv
-VIRTUALENV_DIR=tmp
+VIRTUALENV_DIR=venv
 
 # Cleanable files and directories with the --clean option
 CLEANABLE="
     build
-	tmp"
+	venv"
 
 # extra  arguments passed to pip
 PIP_EXTRA_ARGS=" "
@@ -50,6 +50,12 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
 CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
+# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
+if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then
+    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty "
+fi
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi"
+
 
 ################################
 # Set the quiet flag to empty if not defined 
@@ -78,10 +84,14 @@ cli_help() {
     echo "  usage: ./configure [options]"
     echo
     echo The default is to configure for regular use. Use --dev for development.
+    echo Use the --init option if starting a new project and the project
+    echo dependencies are not available on thirdparty.aboutcode.org/pypi/
+    echo and requirements.txt and/or requirements-dev.txt has not been generated.
     echo
     echo The options are:
     echo " --clean: clean built and installed files and exit."
     echo " --dev:   configure the environment for development."
+    echo " --init:  pull dependencies from PyPI. Used when first setting up a project."
     echo " --help:  display this help message and exit."
     echo
     echo By default, the python interpreter version found in the path is used.
@@ -155,12 +165,22 @@ install_packages() {
 # Main command line entry point
 CFG_DEV_MODE=0
 CFG_REQUIREMENTS=$REQUIREMENTS
-
-case "$CLI_ARGS" in
-    --help)    cli_help;;
-    --clean)   clean;;
-    --dev)     CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
+NO_INDEX="--no-index"
+
+# We are using getopts to parse option arguments that start with "-"
+while getopts :-: optchar; do
+    case "${optchar}" in
+        -)
+            case "${OPTARG}" in
+                help  ) cli_help;;
+                clean ) clean;;
+                dev   ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
+                init  ) NO_INDEX="";;
+            esac;;
 esac
+done
+
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX"
 
 create_virtualenv "$VIRTUALENV_DIR"
 install_packages "$CFG_REQUIREMENTS"

diff --git a/configure.bat b/configure.bat
@@ -24,14 +24,14 @@
 @rem ################################
 
 @rem # Requirement arguments passed to pip and used by default or with --dev.
-set "REQUIREMENTS=--editable .[full]"
-set "DEV_REQUIREMENTS=--editable .[full,testing]"
+set "REQUIREMENTS=--editable .[full] --constraint requirements.txt"
+set "DEV_REQUIREMENTS=--editable .[full,testing] --constraint requirements.txt --constraint requirements-dev.txt"
 
 @rem # where we create a virtualenv
-set "VIRTUALENV_DIR=tmp"
+set "VIRTUALENV_DIR=venv"
 
 @rem # Cleanable files and directories to delete with the --clean option
-set "CLEANABLE=build tmp"
+set "CLEANABLE=build venv"
 
 @rem # extra  arguments passed to pip
 set "PIP_EXTRA_ARGS= "
@@ -47,6 +47,16 @@ set CFG_ROOT_DIR=%~dp0
 set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts"
 
 
+@rem ################################
+@rem # Thirdparty package locations and index handling
+if exist ""%CFG_ROOT_DIR%\thirdparty"" (
+    set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty "
+)
+
+set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG%
+@rem ################################
+
+
 @rem ################################
 @rem # Set the quiet flag to empty if not defined
 if not defined CFG_QUIET (
@@ -58,19 +68,26 @@ if not defined CFG_QUIET (
 @rem # Main command line entry point
 set CFG_DEV_MODE=0
 set "CFG_REQUIREMENTS=%REQUIREMENTS%"
+set "NO_INDEX=--no-index"
 
+:again
+if not "%1" == "" (
 if "%1" EQU "--help"   (goto cli_help)
 if "%1" EQU "--clean"  (goto clean)
 if "%1" EQU "--dev"    (
     set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%"
     set CFG_DEV_MODE=1
 )
-if "%1" EQU "--python" (
-    echo "The --python option is now DEPRECATED. Use the PYTHON_EXECUTABLE environment"
-    echo "variable instead. Run configure --help for details."
-    exit /b 0
+    if "%1" EQU "--init"   (
+        set "NO_INDEX= "
+    )
+    shift
+    goto again
 )
 
+set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%"
+
+
 @rem ################################
 @rem # find a proper Python to run
 @rem # Use environment variables or a file if available.
@@ -142,6 +159,9 @@ if %ERRORLEVEL% neq 0 (
     %PIP_EXTRA_ARGS% ^
     %CFG_REQUIREMENTS%
 
+@rem # Create junction to bin to have the same directory between linux and windows
+mklink /J %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts
+
 if %ERRORLEVEL% neq 0 (
     exit /b %ERRORLEVEL%
 )
@@ -156,10 +176,14 @@ exit /b 0
     echo "  usage: configure [options]"
     echo " "
     echo The default is to configure for regular use. Use --dev for development.
+    echo Use the --init option if starting a new project and the project
+    echo dependencies are not available on thirdparty.aboutcode.org/pypi/
+    echo and requirements.txt and/or requirements-dev.txt has not been generated.
     echo " "
     echo The options are:
     echo " --clean: clean built and installed files and exit."
     echo " --dev:   configure the environment for development."
+    echo " --init:  pull dependencies from PyPI. Used when first setting up a project."
     echo " --help:  display this help message and exit."
     echo " "
     echo By default, the python interpreter version found in the path is used.
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,6 +12,7 @@ @@
     /Lib
     /pip-selfcheck.json
     /tmp
+    /venv
     .Python
     /include
     /Include
@@ Expand Down @@