From e55d5c9d16edee96e115b0b634943ca3c5fd1640 Mon Sep 17 00:00:00 2001 From: Todd Pihl Date: Tue, 3 Feb 2026 10:51:04 -0500 Subject: [PATCH 1/3] Submission Update --- SubmissionResetGUI.py | 277 +++++++++++++++++++++++++++++++++++++++++ SubmissionResetPyQt.py | 204 ++++++++++++++++++++++++++++++ 2 files changed, 481 insertions(+) create mode 100644 SubmissionResetGUI.py create mode 100644 SubmissionResetPyQt.py diff --git a/SubmissionResetGUI.py b/SubmissionResetGUI.py new file mode 100644 index 0000000..e39fabb --- /dev/null +++ b/SubmissionResetGUI.py @@ -0,0 +1,277 @@ +from dash import html, dcc, dash_table, Output, Input, State +import dash +import dash_bootstrap_components as dbc +import requests +import pandas as pd +import io +from datetime import datetime, timezone +from pytz import timezone as tz +import os + +####################################### +# # +# App Definition # +# # +####################################### + +external_stylesheets = [ + { "href": "https://fonts.googleapis.com/css2?" + "family=Lato:wght@400;700&display=swap", + "rel": "stylesheet", + }, + dbc.themes.BOOTSTRAP +] + +app = dash.Dash( + __name__, + external_stylesheets=external_stylesheets, + suppress_callback_exceptions=True, + prevent_initial_callbacks=True, + update_title="Updating..." +) +app.title ="DH Dashboard" + +############################################ +# # +# Styles # +# # +############################################ + +CONTENT_STYLE = { + "margin-left": "18rem", + "margin-right": "12rem", + "padding": "2rem 1 rem" +} + + + + +####################################### +# # +# Subroutines # +# # +####################################### + + +def dhAPICreds(tier): + url = None + token = None + if tier == 'production': + url = 'https://hub.datacommons.cancer.gov/api/graphql' + token = os.getenv('PRODAPI') + elif tier == 'stage': + url = 'https://hub-stage.datacommons.cancer.gov/api/graphql' + token = os.getenv('STAGEAPI') + elif tier == 'qa': + url = 'https://hub-qa.datacommons.cancer.gov/api/graphql' + token = os.getenv('QAAPI') + elif tier == 'qa2': + url = 'https://hub-qa2.datacommons.cancer.gov/api/graphql' + token = os.getenv('QA2API') + elif tier == 'dev': + url = 'https://hub-dev.datacommons.cancer.gov/api/graphql' + token = os.getenv('DEVAPI') + elif tier == 'dev2': + url = 'https://hub-dev2.datacommons.cancer.gov/api/graphql' + token = os.getenv('DEV2API') + elif tier == 'localtest': + url = 'https://this.is.a.test/url/graphql' + token = os.getenv('LOCALTESTAPI') + return {'url': url, 'token': token} + + +def dhApiQuery(url, apitoken, query, variables=None): + headers = {"Authorization": f"Bearer {apitoken}"} + try: + if variables is None: + result = requests.post(url=url, headers=headers, json={"query": query}) + else: + result = requests.post(url=url, headers=headers, json={"query": query, "variables": variables}) + if result.status_code == 200: + return result.json() + else: + return (f"Status Code: {result.status_code}\n{result.content}") + except requests.exceptions.HTTPError as e: + return (f"HTTPError: {e}") + + +def getSubmissionData(tier): + tier = tier.lower() + creds = dhAPICreds(tier) + list_sub_query = """ + query ListSubmissions($status: [String]!){ + listSubmissions(status: $status){ + submissions{ + _id + name + submitterID + submitterName + studyAbbreviation + studyID + dbGaPID + createdAt + updatedAt + metadataValidationStatus + fileValidationStatus + status + } + } + } """ + + variables = {"status": ["New", "In Progress"]} + results = dhApiQuery(creds['url'], creds['token'], list_sub_query, variables) + return results + + + +def elapsedTime(submission_df): + days = [] + for index, row in submission_df.iterrows(): + temp = row['updatedAt'].split('T') + update = datetime.strptime(temp[0], '%Y-%m-%d') + update = update.replace(tzinfo=tz('UTC')) + now = datetime.now(timezone.utc) + diff = (now - update).days + days.append(diff) + submission_df.insert(8,'inactiveDays',days,True) + return submission_df + +############################################ +# # +# Components # +# # +############################################ + +dropdown = html.Div([ + html.H2("Submission Update Tool", className="display-4"), + html.Hr(), + html.Div( + className='tierdropdown', + children=[ + dcc.Dropdown( + id='tierselector', + options=[ + {'label': 'Production', 'value': 'production'}, + {'label': 'Stage', 'value': 'stage'}, + {'label': 'QA', 'value': 'qa'}, + {'label': 'QA2', 'value': 'qa2'}, + {'label': 'Dev', 'value': 'dev'}, + {'label': 'Dev2', 'value': 'dev2'} + ], + placeholder='Select a tier', + multi=False, + style={'backgroundcolor':'1E1E1E'}, + ), + dcc.Store(id="submissionstore") + ] + ) +]) + + +messages = dcc.Textarea( + id='messagearea', + value='Messages Go Here', + style={'width':'100%', 'height': 300} + ) + +updateButton = html.Button('Update Selected', id='updatethis', n_clicks=0) + + +tableheader = html.Div([ + html.Hr(), + html.H2("Study Information", id='studytabletitle'), + html.Hr() +], + style=CONTENT_STYLE) + +submissioncontent = html.Div( + [ + html.Div(dbc.Spinner(html.Div(id="submissioncontentspinner"), color="primary")), + html.Div(id="submissioncontent", style=CONTENT_STYLE) + ] +) + +#################################### +# # +# Layouts # +# # +#################################### + +app.layout = html.Div([ + html.Div( + [dropdown] + ), + html.Div([tableheader, submissioncontent]), + html.Div([updateButton, messages]) +]) + +#app.layout = html.Div( +# children=[dropdown, +# tableheader, +# submissioncontent] +#) + + +#################################### +# # +# Callbacks # +# # +#################################### + +################### Table callbacks ################# +@app.callback( + Output('submissioncontent', 'children'), + Input('tierselector', 'value') +) + +def populateSubmissionsTable(tierselector): + subjson = getSubmissionData(tierselector) + sub_df = pd.DataFrame(subjson['data']['listSubmissions']['submissions']) + sub_df = elapsedTime(sub_df) + sub_df.reset_index().to_json(orient='split') + + return dash_table.DataTable( + data=sub_df.to_dict('records'), + columns=[{"name": e, "id": e} for e in (sub_df.columns)], + row_selectable="multi", + style_table={'overflowX':'auto'}, + style_cell={'overflow':'hidden', 'textOverflow':'ellipsis', 'maxWidth':10, 'textAlign':'center'}, + style_data={'color':'black', 'backgroundColor':'white'}, + style_data_conditional=[{'if':{'row_index':'odd'}, 'backgroundColor': 'rgb(220,220,220)'}], + style_header={'backgroundColor': 'rgb(210,210,210)', 'color':'black', 'fontWeight':'bold', 'textAlign':'center'}, + tooltip_data=[ + { + column:{'value': str(value), 'type':'markdown'} + for column, value in row.items() + } for row in sub_df.to_dict('records') + ], + tooltip_duration=None, + export_format="csv" + ) + +######################## button callback ********************** +# +# https://community.plotly.com/t/how-to-get-the-data-of-the-selected-rows-of-dash-table-experiments/8439 +# +@app.callback( + [ + Input('updatethis', 'n_clicks'), + Input('submissioncontent', 'rows'), + Input('submissioncontent', 'selected_row_indices') + ] +) + +def updateSubmissions(n_clicks,rows, selected_rows_indicies): + for i in selected_rows_indicies: + print(f"Row {str(i)} is {rows[i]}") + + +#################################### +# # +# Run Program # +# # +#################################### + + +if __name__ == "__main__": + app.run(port=8050, debug=True) \ No newline at end of file diff --git a/SubmissionResetPyQt.py b/SubmissionResetPyQt.py new file mode 100644 index 0000000..e9a665d --- /dev/null +++ b/SubmissionResetPyQt.py @@ -0,0 +1,204 @@ +# +# https://www.pythonguis.com/pyqt6-tutorial/ +# https://zetcode.com/pyqt6/ +# +from PyQt6.QtCore import QSize, Qt, QAbstractTableModel, QModelIndex +from PyQt6.QtGui import QAction, QIcon, QKeySequence +from PyQt6.QtWidgets import( + QApplication, + QCheckBox, + QLabel, + QMainWindow, + QStatusBar, + QToolBar, + QVBoxLayout, + QComboBox, + QWidget, + QHBoxLayout, + QTableView +) + +import requests +import os +import sys +import pandas as pd + + +class PandasModel(QAbstractTableModel): + # https://www.pythonguis.com/tutorials/pyqt6-qtableview-modelviews-numpy-pandas/ + # https://doc.qt.io/qtforpython-6/examples/example_external_pandas.html + + def __init__(self, dataframe:pd.DataFrame, parent=None): + #super().__init__() + QAbstractTableModel.__init__(self, parent) + self._dataframe = dataframe + + #def data(self, index, role): + def data(self, index:QModelIndex, role=Qt.ItemDataRole): + if not index.isValid(): + return None + if role == Qt.ItemDataRole.DisplayRole: + return str(self._dataframe.iloc[index.row(), index.column()]) + return None + #if role == Qt.ItemDataRole.DisplayRole: + # value = self.data[index.row()][index.column()] + # return str(value) + + + def headerData(self, section: int, orientation: Qt.Orientation, role: Qt.ItemDataRole): + if role == Qt.ItemDataRole.DisplayRole: + if orientation == Qt.Orientation.Horizontal: + return str(self._dataframe.columns[section]) + if orientation == Qt.Orientation.Vertical: + return str(self._dataframe.index[section]) + return None + + def rowCount(self, parent=QModelIndex()) -> int: + if parent == QModelIndex(): + return self._dataframe.shape[0] + # return len(self._dataframe) + #return len(self._data) + #return self._data.shape[0] + return 0 + + def columnCount(self, parent=QModelIndex()) -> int: + if parent == QModelIndex(): + return self._dataframe.shape[1] + # return len(self._dataframe.columns) + #return len(self._data[0]) + #return self._data.shape[1] + return 0 + + +class MainWindow(QMainWindow): + def __init__(self): + super().__init__() + self.setWindowTitle("CRDC Data Hub Submission Reset") + self.resize(400,200) + + self.subtable = QTableView() + + + #self._createMenuBar() + + tier_layout = QHBoxLayout() + table_layout = QVBoxLayout() + main_layout = QVBoxLayout() + + tier_label = QLabel("Select a tier: ") + table_label = QLabel("Submissions") + + tier_combobox = QComboBox() + tier_combobox.addItems(["","Production", "Stage", "QA", "Dev"]) + tier_combobox.currentIndexChanged.connect(self.tier_index_changed) + tier_combobox.currentTextChanged.connect(self.tier_text_changed) + tier_combobox.currentTextChanged.connect(self.getSubmissionData) + + tier_layout.addWidget(tier_label) + tier_layout.addWidget(tier_combobox) + + subtable = QTableView() + subtable.horizontalHeader().setStretchLastSection(True) + subtable.setAlternatingRowColors(True) + subtable.setSelectionBehavior(QTableView.selectRow) + + widget = QWidget() + widget.setLayout(tier_layout) + self.setCentralWidget(widget) + + + + def tier_index_changed(self, index): + print(index) + + def tier_text_changed(self, text): + print(text) + + + + + + def dhAPICreds(self,tier): + url = None + token = None + if tier == 'production': + url = 'https://hub.datacommons.cancer.gov/api/graphql' + token = os.getenv('PRODAPI') + elif tier == 'stage': + url = 'https://hub-stage.datacommons.cancer.gov/api/graphql' + token = os.getenv('STAGEAPI') + elif tier == 'qa': + url = 'https://hub-qa.datacommons.cancer.gov/api/graphql' + token = os.getenv('QAAPI') + elif tier == 'qa2': + url = 'https://hub-qa2.datacommons.cancer.gov/api/graphql' + token = os.getenv('QA2API') + elif tier == 'dev': + url = 'https://hub-dev.datacommons.cancer.gov/api/graphql' + token = os.getenv('DEVAPI') + elif tier == 'dev2': + url = 'https://hub-dev2.datacommons.cancer.gov/api/graphql' + token = os.getenv('DEV2API') + elif tier == 'localtest': + url = 'https://this.is.a.test/url/graphql' + token = os.getenv('LOCALTESTAPI') + return {'url': url, 'token': token} + + + + def dhApiQuery(self, url, apitoken, query, variables=None): + headers = {"Authorization": f"Bearer {apitoken}"} + try: + if variables is None: + result = requests.post(url=url, headers=headers, json={"query": query}) + else: + result = requests.post(url=url, headers=headers, json={"query": query, "variables": variables}) + if result.status_code == 200: + return result.json() + else: + return (f"Status Code: {result.status_code}\n{result.content}") + except requests.exceptions.HTTPError as e: + return (f"HTTPError: {e}") + + + def getSubmissionData(self, tier): + tier = tier.lower() + print(f"Getting submission data for {tier}") + creds = self.dhAPICreds(tier) + list_sub_query = """ + query ListSubmissions($status: [String]!){ + listSubmissions(status: $status){ + submissions{ + _id + name + submitterID + submitterName + studyAbbreviation + studyID + dbGaPID + createdAt + updatedAt + metadataValidationStatus + fileValidationStatus + status + } + } + } + """ + subvariables = {"status": ["New", "In Progress"]} + subres = self.dhApiQuery(url=creds['url'], apitoken=creds['token'], query=list_sub_query, variables=subvariables) + #print(subres) + sub_df = pd.DataFrame(subres['data']['listSubmissions']['submissions']) + print(sub_df.head()) + + + + + + +app = QApplication([]) + +mainWindow = MainWindow() +mainWindow.show() + +app.exec() \ No newline at end of file From cca32a97a7f70a753a0b738a7fe95da74f9f1343 Mon Sep 17 00:00:00 2001 From: Todd Pihl Date: Tue, 3 Feb 2026 15:25:53 -0500 Subject: [PATCH 2/3] Submission Update and README --- README.md | 21 ++++- SubmissionResetGUI.py | 122 ++++++++++++++++-------- SubmissionResetPyQt.py | 204 ----------------------------------------- 3 files changed, 102 insertions(+), 245 deletions(-) delete mode 100644 SubmissionResetPyQt.py diff --git a/README.md b/README.md index 834c949..789b70e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # DataHubAPIDemo Demonstration scripts and notebooks showing how to use the Data Hub API for data submissions, reporting, and other tasks. -**Note:** All of these scripts will require a Data Hub API key in order to use. Instructions for obtaining an API token can be found in the [data submission documentation](https://datacommons.cancer.gov/data-submission-instructions) +**Note:** All of these scripts will require a Data Hub API key in order to use. Instructions for obtaining an API token can be found in the [data submission documentation](https://datacommons.cancer.gov/data-submission-instructions). For security reasons, these tokens should be stored as environment variables on your system. The scripts expect production and stage API keys to be stored in the environment variables PRODAPI and STAGEAPI, respectively. ## DataHubAPIDemo.ipynb This Jupyter notebooks walks through a basic example of how to do a CRDC submission using the Data Hub APIs. @@ -23,7 +23,10 @@ This notebook covers several queries that can provide more detailed information - Retrieving a populated configuration file for use in uploading data files with the CLI Upload Tool ## SubmissionReportDashboard.py -This is a Python Dash application that uses the APIs to create a personal dashboard of your submissions. +This is a Python Dash application that uses the APIs to create a personal dashboard of your submissions. To use this script, run the script (# python3 SubmissionReportDashboard.py), then launch a browser and navigate to http://localhost:8050. + +**Required Python Libraries** +dash, dash_bootstrap_components, plotly, requests, pandas, datetime, pytz ## ShinyDashboard.py @@ -31,4 +34,16 @@ Simialr to the SubmissionReportDashboard only uses Python Shiny instead of Dash. ## WarningAggregator.ipynb and WarningAggregator.py -Submissions that are inactive for extended periods of time start generating warning emails and after 180 days get deleted. The remedy to this situation is to log into the Submission Portal and look at the submission. However, this gets burdensome if there are a large number of submissions to check. This script (also in notebook form) will query for all the submissions that are either New or In Progress and will request information from each of them. This re-sets the inatvitiy timer. + + + +## SubmissionReset.py +This script resets the inactivity timer for all of your **New** or **In Progress submissions** to the current date. + +## SubmissionResetGUI.py +This is a graphical version (Python Dash) of the *SubmissionReset.py* script. Select a tier from the drop-down and a table of your current New and In Progress submissions will be generated. Select the submissions you wish to reset the inactivity timer on and then click on the *Reset Time on Selected Submissions* button below the table. Each submission selected will be reset to the current date. + +To use, run the script ($ python3 SubmissionResetGUI.py) and then bring up a browser and navigate to http://localhost:8050. + +**Required Python Libraries** +dash, dash_bootstrap_components, requests, pandas, datetime, pytz \ No newline at end of file diff --git a/SubmissionResetGUI.py b/SubmissionResetGUI.py index e39fabb..0f1d5ba 100644 --- a/SubmissionResetGUI.py +++ b/SubmissionResetGUI.py @@ -4,6 +4,7 @@ import requests import pandas as pd import io +import json from datetime import datetime, timezone from pytz import timezone as tz import os @@ -136,15 +137,48 @@ def elapsedTime(submission_df): submission_df.insert(8,'inactiveDays',days,True) return submission_df + +def updateSubmissionClock(subid, tier): + getSubmissionQuery = """ + query GetSubmissions( + $id: ID! + ){ + getSubmission(_id:$id){ + _id + name + dataCommons + } + } + + """ + vars = {"id": subid} + creds = dhAPICreds(tier) + #print(f"Vars: {vars}\n Creds: {creds}\n") + updatejson = dhApiQuery(creds['url'], creds['token'], getSubmissionQuery, vars) + #updatejson = {} + return updatejson + + + + ############################################ # # # Components # # # ############################################ +pageheader = html.Div([ + html.Div( + html.H1('Submission Update Tool'), + style={'textAlign':'center'} + ), + html.Div( + children='A tool to reset the submission clock', + style={'textAlign':'center'} + ) +]) + dropdown = html.Div([ - html.H2("Submission Update Tool", className="display-4"), - html.Hr(), html.Div( className='tierdropdown', children=[ @@ -162,32 +196,25 @@ def elapsedTime(submission_df): multi=False, style={'backgroundcolor':'1E1E1E'}, ), - dcc.Store(id="submissionstore") ] ) ]) -messages = dcc.Textarea( - id='messagearea', - value='Messages Go Here', - style={'width':'100%', 'height': 300} - ) +errormessages = html.Div(id='printhere', style={'whiteSpace':'pre-line'}) -updateButton = html.Button('Update Selected', id='updatethis', n_clicks=0) +updateButton = html.Button('Reset Time on Selected Submissions', id='updatethis', n_clicks=0) tableheader = html.Div([ - html.Hr(), - html.H2("Study Information", id='studytabletitle'), - html.Hr() + html.H2("Your current Submissions", style={'textAlign':'center'}), ], style=CONTENT_STYLE) submissioncontent = html.Div( [ html.Div(dbc.Spinner(html.Div(id="submissioncontentspinner"), color="primary")), - html.Div(id="submissioncontent", style=CONTENT_STYLE) + html.Div(id="submissioncontent") ] ) @@ -198,19 +225,17 @@ def elapsedTime(submission_df): #################################### app.layout = html.Div([ - html.Div( - [dropdown] - ), - html.Div([tableheader, submissioncontent]), - html.Div([updateButton, messages]) + pageheader, + html.Div([dropdown]), + html.Hr(), + html.Div([tableheader]), + html.Hr(), + html.Div([submissioncontent]), + html.Div([updateButton]), + html.Div([errormessages]), + dcc.Store(id='substore') ]) -#app.layout = html.Div( -# children=[dropdown, -# tableheader, -# submissioncontent] -#) - #################################### # # @@ -218,22 +243,33 @@ def elapsedTime(submission_df): # # #################################### -################### Table callbacks ################# +################## Datastore callbacks ###############3 @app.callback( - Output('submissioncontent', 'children'), + Output('substore', 'data', allow_duplicate=True), Input('tierselector', 'value') ) - -def populateSubmissionsTable(tierselector): +def updateSubstore(tierselector): subjson = getSubmissionData(tierselector) sub_df = pd.DataFrame(subjson['data']['listSubmissions']['submissions']) sub_df = elapsedTime(sub_df) - sub_df.reset_index().to_json(orient='split') + return sub_df.reset_index().to_json(orient='split') +################### Table callbacks ################# + +@app.callback( + Output('submissioncontent', 'children'), + Input('substore', 'modified_timestamp'), + State('substore', 'data') +) +def populateSubmissionsTable2(modified_timestamp, substore): + sub_df = pd.read_json(io.StringIO(substore), orient='split') return dash_table.DataTable( + id = 'subtable', data=sub_df.to_dict('records'), columns=[{"name": e, "id": e} for e in (sub_df.columns)], row_selectable="multi", + sort_action='native', + sort_mode='multi', style_table={'overflowX':'auto'}, style_cell={'overflow':'hidden', 'textOverflow':'ellipsis', 'maxWidth':10, 'textAlign':'center'}, style_data={'color':'black', 'backgroundColor':'white'}, @@ -248,23 +284,33 @@ def populateSubmissionsTable(tierselector): tooltip_duration=None, export_format="csv" ) - ######################## button callback ********************** # # https://community.plotly.com/t/how-to-get-the-data-of-the-selected-rows-of-dash-table-experiments/8439 # @app.callback( - [ - Input('updatethis', 'n_clicks'), - Input('submissioncontent', 'rows'), - Input('submissioncontent', 'selected_row_indices') - ] + Output('substore', 'data', allow_duplicate=True), + Input('updatethis', 'n_clicks'), + State('subtable', 'selected_rows'), + State('substore', 'data'), + State('tierselector', 'value') ) -def updateSubmissions(n_clicks,rows, selected_rows_indicies): - for i in selected_rows_indicies: - print(f"Row {str(i)} is {rows[i]}") +def updateSubmissions(n_clicks,selected_rows, substore, tierselector): + if n_clicks >= 0: + sub_df = pd.read_json(io.StringIO(substore), orient='split') + selected_df = sub_df.iloc[selected_rows] + collected_res = [] + for index, row in selected_df.iterrows(): + print(row['_id']) + res = updateSubmissionClock(row['_id'], tierselector) + collected_res.append({row['name']: res}) + + subjson = getSubmissionData(tierselector) + sub_df = pd.DataFrame(subjson['data']['listSubmissions']['submissions']) + sub_df = elapsedTime(sub_df) + return sub_df.reset_index().to_json(orient='split') #################################### # # diff --git a/SubmissionResetPyQt.py b/SubmissionResetPyQt.py deleted file mode 100644 index e9a665d..0000000 --- a/SubmissionResetPyQt.py +++ /dev/null @@ -1,204 +0,0 @@ -# -# https://www.pythonguis.com/pyqt6-tutorial/ -# https://zetcode.com/pyqt6/ -# -from PyQt6.QtCore import QSize, Qt, QAbstractTableModel, QModelIndex -from PyQt6.QtGui import QAction, QIcon, QKeySequence -from PyQt6.QtWidgets import( - QApplication, - QCheckBox, - QLabel, - QMainWindow, - QStatusBar, - QToolBar, - QVBoxLayout, - QComboBox, - QWidget, - QHBoxLayout, - QTableView -) - -import requests -import os -import sys -import pandas as pd - - -class PandasModel(QAbstractTableModel): - # https://www.pythonguis.com/tutorials/pyqt6-qtableview-modelviews-numpy-pandas/ - # https://doc.qt.io/qtforpython-6/examples/example_external_pandas.html - - def __init__(self, dataframe:pd.DataFrame, parent=None): - #super().__init__() - QAbstractTableModel.__init__(self, parent) - self._dataframe = dataframe - - #def data(self, index, role): - def data(self, index:QModelIndex, role=Qt.ItemDataRole): - if not index.isValid(): - return None - if role == Qt.ItemDataRole.DisplayRole: - return str(self._dataframe.iloc[index.row(), index.column()]) - return None - #if role == Qt.ItemDataRole.DisplayRole: - # value = self.data[index.row()][index.column()] - # return str(value) - - - def headerData(self, section: int, orientation: Qt.Orientation, role: Qt.ItemDataRole): - if role == Qt.ItemDataRole.DisplayRole: - if orientation == Qt.Orientation.Horizontal: - return str(self._dataframe.columns[section]) - if orientation == Qt.Orientation.Vertical: - return str(self._dataframe.index[section]) - return None - - def rowCount(self, parent=QModelIndex()) -> int: - if parent == QModelIndex(): - return self._dataframe.shape[0] - # return len(self._dataframe) - #return len(self._data) - #return self._data.shape[0] - return 0 - - def columnCount(self, parent=QModelIndex()) -> int: - if parent == QModelIndex(): - return self._dataframe.shape[1] - # return len(self._dataframe.columns) - #return len(self._data[0]) - #return self._data.shape[1] - return 0 - - -class MainWindow(QMainWindow): - def __init__(self): - super().__init__() - self.setWindowTitle("CRDC Data Hub Submission Reset") - self.resize(400,200) - - self.subtable = QTableView() - - - #self._createMenuBar() - - tier_layout = QHBoxLayout() - table_layout = QVBoxLayout() - main_layout = QVBoxLayout() - - tier_label = QLabel("Select a tier: ") - table_label = QLabel("Submissions") - - tier_combobox = QComboBox() - tier_combobox.addItems(["","Production", "Stage", "QA", "Dev"]) - tier_combobox.currentIndexChanged.connect(self.tier_index_changed) - tier_combobox.currentTextChanged.connect(self.tier_text_changed) - tier_combobox.currentTextChanged.connect(self.getSubmissionData) - - tier_layout.addWidget(tier_label) - tier_layout.addWidget(tier_combobox) - - subtable = QTableView() - subtable.horizontalHeader().setStretchLastSection(True) - subtable.setAlternatingRowColors(True) - subtable.setSelectionBehavior(QTableView.selectRow) - - widget = QWidget() - widget.setLayout(tier_layout) - self.setCentralWidget(widget) - - - - def tier_index_changed(self, index): - print(index) - - def tier_text_changed(self, text): - print(text) - - - - - - def dhAPICreds(self,tier): - url = None - token = None - if tier == 'production': - url = 'https://hub.datacommons.cancer.gov/api/graphql' - token = os.getenv('PRODAPI') - elif tier == 'stage': - url = 'https://hub-stage.datacommons.cancer.gov/api/graphql' - token = os.getenv('STAGEAPI') - elif tier == 'qa': - url = 'https://hub-qa.datacommons.cancer.gov/api/graphql' - token = os.getenv('QAAPI') - elif tier == 'qa2': - url = 'https://hub-qa2.datacommons.cancer.gov/api/graphql' - token = os.getenv('QA2API') - elif tier == 'dev': - url = 'https://hub-dev.datacommons.cancer.gov/api/graphql' - token = os.getenv('DEVAPI') - elif tier == 'dev2': - url = 'https://hub-dev2.datacommons.cancer.gov/api/graphql' - token = os.getenv('DEV2API') - elif tier == 'localtest': - url = 'https://this.is.a.test/url/graphql' - token = os.getenv('LOCALTESTAPI') - return {'url': url, 'token': token} - - - - def dhApiQuery(self, url, apitoken, query, variables=None): - headers = {"Authorization": f"Bearer {apitoken}"} - try: - if variables is None: - result = requests.post(url=url, headers=headers, json={"query": query}) - else: - result = requests.post(url=url, headers=headers, json={"query": query, "variables": variables}) - if result.status_code == 200: - return result.json() - else: - return (f"Status Code: {result.status_code}\n{result.content}") - except requests.exceptions.HTTPError as e: - return (f"HTTPError: {e}") - - - def getSubmissionData(self, tier): - tier = tier.lower() - print(f"Getting submission data for {tier}") - creds = self.dhAPICreds(tier) - list_sub_query = """ - query ListSubmissions($status: [String]!){ - listSubmissions(status: $status){ - submissions{ - _id - name - submitterID - submitterName - studyAbbreviation - studyID - dbGaPID - createdAt - updatedAt - metadataValidationStatus - fileValidationStatus - status - } - } - } - """ - subvariables = {"status": ["New", "In Progress"]} - subres = self.dhApiQuery(url=creds['url'], apitoken=creds['token'], query=list_sub_query, variables=subvariables) - #print(subres) - sub_df = pd.DataFrame(subres['data']['listSubmissions']['submissions']) - print(sub_df.head()) - - - - - - -app = QApplication([]) - -mainWindow = MainWindow() -mainWindow.show() - -app.exec() \ No newline at end of file From 59197f692fc474bc4ce1a6779e35f22436ee69a7 Mon Sep 17 00:00:00 2001 From: Todd Pihl Date: Tue, 3 Feb 2026 15:50:27 -0500 Subject: [PATCH 3/3] README update --- README.md | 21 +++++++++++++++++++-- SubmissionResetGUI.py | 1 - 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 789b70e..3023926 100644 --- a/README.md +++ b/README.md @@ -34,16 +34,33 @@ Simialr to the SubmissionReportDashboard only uses Python Shiny instead of Dash. ## WarningAggregator.ipynb and WarningAggregator.py - +Currently, the Submission Portal does not have aggreation for the Warnings that are generated by data updates. This can make it difficult to see if the updates that are about to be applied are correct. These programs will aggregate all the warnings and display them in a paired manner so that it is easier to see what changes are about to be applied. + +Both scripts take the following configurations. In the notebook you will find these in a marked cell, edit that cell to need. For the script, the configurations are provided in a YAML file (see *warning_configs.yml* for an example) + +**subid**: A list of the submission IDs to be checked. Submission IDs can be obtained from the Submission Portal +**severity**: Should be set to 'All' +**nodelist**: A list of the nodes that should be checked for Warnings +**outputdirectory**: A local directory where output can be written +**tier**: The tier to use, shoudl be either 'stage' or 'prod' + +**Script runtime options** + - -c/--configfile: The YAML configuration file + - -v/--verbose: The level of verbosity. Add more v's to be more verbose. ## SubmissionReset.py This script resets the inactivity timer for all of your **New** or **In Progress submissions** to the current date. +**Script runtime options** +- -t/--tier: The tier to check. Should be either 'prod' or 'stage' +- -v/--verbose: The level of verbosity. Add more v's to be more verbose. + ## SubmissionResetGUI.py This is a graphical version (Python Dash) of the *SubmissionReset.py* script. Select a tier from the drop-down and a table of your current New and In Progress submissions will be generated. Select the submissions you wish to reset the inactivity timer on and then click on the *Reset Time on Selected Submissions* button below the table. Each submission selected will be reset to the current date. To use, run the script ($ python3 SubmissionResetGUI.py) and then bring up a browser and navigate to http://localhost:8050. **Required Python Libraries** -dash, dash_bootstrap_components, requests, pandas, datetime, pytz \ No newline at end of file +dash, dash_bootstrap_components, requests, pandas, datetime, pytz + diff --git a/SubmissionResetGUI.py b/SubmissionResetGUI.py index 0f1d5ba..ebcfe47 100644 --- a/SubmissionResetGUI.py +++ b/SubmissionResetGUI.py @@ -4,7 +4,6 @@ import requests import pandas as pd import io -import json from datetime import datetime, timezone from pytz import timezone as tz import os