Getting scraper py310 to deploy properly and work

.dockerignore

@@ -15,4 +15,8 @@ env/
 venv/
 .venv*
 crontab-test
-.vscode/
+.vscode/
+
+cf*
+bin*
+tests*

Dockerfile

@@ -4,7 +4,6 @@ ENV SUPERCRONIC_URL=https://github.com/albertcrowley/supercronic/releases/downlo
     SUPERCRONIC=supercronic-linux-x86 \
     SUPERCRONIC_SHA1SUM=2b5144dee1af0dc07c372c3c45026dd42af81226
 
-RUN pip install --upgrade pip
 ADD requirements.txt .
 
 RUN apt-get update && apt-get install -y \
@@ -43,8 +42,6 @@ RUN apt-get update && apt-get install -y \
     #clean up the apt cache
     && rm -rf /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt --no-cache-dir
 
 RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
     && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
@@ -54,10 +51,11 @@ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key
     && unzip /tmp/chromedriver.zip chromedriver -d /usr/local/bin/
 
 ADD . .
-RUN pip install .
+RUN pip install --upgrade pip && pip install -e .
 
 #see https://docs.cloudfoundry.org/devguide/deploy-apps/push-docker.html
 COPY ./conf/passwd /etc/passwd
+COPY ./conf /usr/local/conf
 
 ENTRYPOINT ["supercronic"]
 

conf/config.yml

@@ -5,4 +5,6 @@ client:
   from_date: "yesterday"
   to_date: "yesterday"
 database:
-  update_old: True
+  update_old: True
+prediction: 
+  model_name: "clf_ajbuckingham_roc_auc.pkl"

crontab

@@ -1,2 +1,2 @@
 # Run every day at 5 AM Eastern Standard Time
-0 10 * * * /usr/local/bin/python3.6 main.py
+0 10 * * * /usr/local/bin/fbo_scraper

pyproject.toml

@@ -3,10 +3,9 @@ requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
-addopts = "--cov=fbo_scraper"
-testpaths = [
-    "tests",
-]
+addopts = "--cov=src"
+testpaths = ["tests"]
+pythonpath = "src"
 
 [tool.mypy]
 mypy_path = "src"
@@ -63,8 +62,8 @@ line-length = 88
 # Allow unused variables when underscore-prefixed.
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 
-# Assume Python 3.7 minimum.
-target-version = "py37"
+# Assume Python 3.10 minimum.
+target-version = "py310"
 
 [tool.ruff.mccabe]
 # Unlike Flake8, default to a complexity level of 10.

setup.cfg

@@ -9,14 +9,10 @@ platforms = unix, linux, osx
 classifiers =
     Programming Language :: Python :: 3 
     Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.7
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
 
 [options]
-packages = 
-    fbo_scraper
+packages = find:
 install_requires = 
     addict
     dill
@@ -28,30 +24,45 @@ install_requires =
     python-json-logger
     PyYAML
     requests
-    scikit-learn
+    scikit-learn==1.2.2
     scipy
     sqlalchemy
     sqlalchemy_utils
     textract
     urllib3
     wget
-python_requires = >=3.7
+python_requires = >=3.10
 package_dir =
     =src
-zip_safe = no
+zip_safe = False
+include_package_data = True
+
 
 [options.extras_require]
 testing =
+    fpdf
+    python-docx
     pytest
     pytest-cov
     mypy
+    requests_mock
+    reportlab
     ruff
-    tox
+
+[options.package_data]
+mypkg =
+    *.pkl
+
+
+[options.packages.find]
+where=src
+exclude =
+    cf*
+    bin*
+    docs*
+
 
 [options.entry_points]
 console_scripts =
     fbo_scraper = fbo_scraper.main:actual_main
 
-[options.package_data]
-fbo_scraper = py.typed
-