dynamic objectstore password

config.ini

@@ -2,7 +2,8 @@
 VERSION=2.0
 AUTHURL=https://identity.stack.cloudvps.com/v2.0
 USER=extern_dataservices
-PASSWORD= **Use export OBJECTSTORE_PASSWORD=###########**
+TENANT_PASSWORD_NAME=EXTERN_DATASERVICES_PASSWORD
+PASSWORD= **Use export EXTERN_DATASERVICES_PASSWORD=###########**
 TENANT_NAME=extern_dataservices
 TENANT_ID=356c76835e424b968ed6d654c51204f0
 REGION_NAME=NL

src/datapunt_processing/extract/download_from_objectstore.py

@@ -2,9 +2,13 @@
 import os
 import argparse
 import logging
+
+from pathlib import Path
+
 from datapunt_processing.helpers.connections import objectstore_connection
 from datapunt_processing.helpers.files import create_dir_if_not_exists
 
+
 logging.basicConfig(level=logging.INFO,
                     format='%(asctime)s %(levelname)-8s %(message)s',
                     datefmt='%a, %d %b %Y %H:%M:%S')
@@ -41,6 +45,11 @@ def get_full_container_list(connection, container, **kwargs):
     return seed
 
 
+def file_exists(target):
+    target = Path(target)
+    return target.is_file()
+
+
 def download_container(connection, container, prefix, output_folder):
     """
     Download file from objectstore.
@@ -55,9 +64,26 @@ def download_container(connection, container, prefix, output_folder):
     """
     target_dir = os.path.join(output_folder, prefix)
     create_dir_if_not_exists(target_dir)
+
     content = get_full_container_list(connection, container['name'], prefix=prefix)
-    # print(content)
+
     for obj in content:
+        # check if object type is not application or dir, or a "part" file
+        if obj['content_type'] == 'application/directory':
+            logger.info('skipping dir')
+            continue
+
+        if 'part' in obj['name']:
+            logger.info('skipping part')
+            continue
+
+        # target filename of object
+        target_filename = os.path.join(target_dir, obj['name'])
+
+        if file_exists(target_filename):
+            logger.info('skipping %s, file already exists', target_filename)
+            continue
+
         if obj['content_type'] != 'application/directory':
             target_filename = os.path.join(output_folder, obj['name'])
             with open(target_filename, 'wb') as new_file:
@@ -78,7 +104,7 @@ def download_containers(connection, prefixes, output_folder):
     Result:
         Loops through download_container function for each prefix (=folder)
     """
-    logger.debug('Checking local data directory exists and is empty')
+    logger.info('Checking local data directory exists and is empty')
     if not os.path.exists(output_folder):
         raise Exception('Local data directory does not exist.')
 
@@ -87,8 +113,7 @@ def download_containers(connection, prefixes, output_folder):
 
     logger.info('Downloading containers ...')
     prefixes = prefixes.split(',')
-    print(containers)
-    print(prefixes)
+
     for container in containers:
         for prefix in prefixes:
             download_container(connection, container, prefix, output_folder)
@@ -100,7 +125,10 @@ def parser():
     Download files from the objectstore:
     ``download_from_objectstore config.ini objectstore aanvalsplan_schoon/crow,aanvalsplan_schoon/mora data``
 
-    Use ``export OBJECTSTORE_PASSWORD=**********`` to add the password to your environment before running this command script.
+    Choose a PASSWORD_NAME to name the objectstore password. Add this to the config.ini file:
+    ``TENANT_PASSWORD_NAME=PASSWORD_NAME``
+
+    Use ``export PASSWORD_NAME=**********`` to add the password to your environment before running this command script.
     """
 
     parser = argparse.ArgumentParser(

src/datapunt_processing/helpers/connections.py

@@ -125,16 +125,18 @@ def objectstore_connection(config_full_path, config_name, print_config_vars=None
         An objectstore connection session.
       """
 
-    assert os.environ['OBJECTSTORE_PASSWORD']
-
     config = get_config(config_full_path)
 
+    tenant_password_name = config.get(config_name, 'TENANT_PASSWORD_NAME')
+
+    assert os.environ[tenant_password_name]
+
     if print_config_vars:
          logger.info('config variables.. :{}'.format(OBJECTSTORE))
 
     conn = Connection(authurl=config.get(config_name, 'AUTHURL'),
                       user=config.get(config_name, 'USER'),
-                      key=os.environ['OBJECTSTORE_PASSWORD'],
+                      key=os.environ[tenant_password_name],
                       tenant_name=config.get(config_name, 'TENANT_NAME'),
                       auth_version=config.get(config_name, 'VERSION'),
                       os_options={'tenant_id': config.get(config_name, 'TENANT_ID'),

src/datapunt_processing/load/load_file_to_objectstore.py

@@ -106,6 +106,8 @@ def parser():
     **Write a file to a container on the objectstore.**
 
     Use ENV:
+    Choose a PASSWORD_NAME to name the objectstore password. Add this to the config.ini file:
+    ``TENANT_PASSWORD_NAME=PASSWORD_NAME``
         ``export OBJECTSTORE_PASSWORD=**********`` to add the password to your environment before running this command script.
 
     Example commandline code: