From 726ba3c68fe2e241a2cae2920546a8a6e31aabe5 Mon Sep 17 00:00:00 2001 From: Aditya Pareek <119134401+LOGANBLUE1@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:46:18 +0530 Subject: [PATCH 1/3] added final changes --- .../java/zingg/spark/core/model/SparkFeatureCreators.java | 6 +++++- .../main/java/zingg/spark/core/model/SparkMLPipeline.java | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/spark/core/src/main/java/zingg/spark/core/model/SparkFeatureCreators.java b/spark/core/src/main/java/zingg/spark/core/model/SparkFeatureCreators.java index db0a4c2e7..3a3f87496 100644 --- a/spark/core/src/main/java/zingg/spark/core/model/SparkFeatureCreators.java +++ b/spark/core/src/main/java/zingg/spark/core/model/SparkFeatureCreators.java @@ -33,12 +33,16 @@ public SparkFeatureCreators(Map> featurers, M for (SimFunction sf : sfList) { String outputCol = columnHelper.getColumnName(fd.fieldName, sf.getName(), count); columnHelper.getColumnsAdded().add(outputCol); - transformers.add(new SparkTransformer(fd.fieldName, new SparkSimFunction(sf), outputCol)); + transformers.add(createTransformer(sf, fd.fieldName, outputCol)); count++; } } } + protected SparkTransformer createTransformer(SimFunction sf, String fieldName, String outputCol) { + return new SparkTransformer(fieldName, new SparkSimFunction(sf), outputCol); + } + public void register(SparkSession session) { for (SparkTransformer t : transformers) { t.register(session); diff --git a/spark/core/src/main/java/zingg/spark/core/model/SparkMLPipeline.java b/spark/core/src/main/java/zingg/spark/core/model/SparkMLPipeline.java index c9c2f0dbd..3524e6bbb 100644 --- a/spark/core/src/main/java/zingg/spark/core/model/SparkMLPipeline.java +++ b/spark/core/src/main/java/zingg/spark/core/model/SparkMLPipeline.java @@ -32,7 +32,6 @@ import zingg.common.core.model.ModelColumnHelper; import zingg.common.core.model.ModelGrid; import zingg.spark.client.SparkFrame; -import zingg.spark.core.similarity.SparkTransformer; public class SparkMLPipeline implements Serializable { @@ -44,7 +43,7 @@ public class SparkMLPipeline implements Serializable { private LogisticRegression lr; private Transformer transformer; private VectorValueExtractor vve; - protected ModelColumnHelper columnHelper; + private ModelColumnHelper columnHelper; public SparkMLPipeline() { } From a43ceb941e694086b415d5a72f44e6bd6305ce76 Mon Sep 17 00:00:00 2001 From: Aditya Pareek <119134401+LOGANBLUE1@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:48:09 +0530 Subject: [PATCH 2/3] Removed isNative --- .../core/similarity/function/SimFunction.java | 5 - scripts/zingg-debug.sh | 106 ++++++++++++++++++ 2 files changed, 106 insertions(+), 5 deletions(-) create mode 100755 scripts/zingg-debug.sh diff --git a/common/core/src/main/java/zingg/common/core/similarity/function/SimFunction.java b/common/core/src/main/java/zingg/common/core/similarity/function/SimFunction.java index d14e3bf8a..768da3d40 100644 --- a/common/core/src/main/java/zingg/common/core/similarity/function/SimFunction.java +++ b/common/core/src/main/java/zingg/common/core/similarity/function/SimFunction.java @@ -10,7 +10,6 @@ public abstract class SimFunction implements Serializable{ //extends UDF2 Date: Wed, 29 Jul 2026 10:49:05 +0530 Subject: [PATCH 3/3] - --- scripts/zingg-debug.sh | 106 ----------------------------------------- 1 file changed, 106 deletions(-) delete mode 100755 scripts/zingg-debug.sh diff --git a/scripts/zingg-debug.sh b/scripts/zingg-debug.sh deleted file mode 100755 index e6ee993f3..000000000 --- a/scripts/zingg-debug.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash - -# Debug launcher for the Spark-based Zingg client. -# Attaches a JDWP debugger on port 5005 (suspend=y by default — change to -# suspend=n to let the process start without waiting for the debugger). -# -# Prerequisites: ZINGG_HOME and SPARK_HOME must be set (or export them below). -# The assembled JAR is expected at $ZINGG_HOME/zingg-0.7.0.jar. -# Override JAR_PATH to point at a different build output. - -# ---------- paths ---------------------------------------------------------- - -JAR_PATH="${JAR_PATH:-$ZINGG_HOME/zingg-0.7.0.jar}" -ZINGG_JARS="$JAR_PATH:$ZINGG_HOME/thirdParty/lib/secondstring.jar:$ZINGG_HOME/thirdParty/lib/py4j0.10.9.jar" - -EMAIL=zingg@zingg.ai -LICENSE=zinggLicense.txt - -log4j_setting="-Dlog4j2.configurationFile=file:log4j2.properties" - -# JDWP debug port — suspend=y means spark-submit blocks until the debugger connects -DEBUG_PORT="${DEBUG_PORT:-5005}" -DEBUG_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:${DEBUG_PORT}" - -# ---------- arg parsing (same contract as zingg.sh) ------------------------ - -export SESSION_TYPE="PY4J" -POSITIONAL_ARGS=() - -while [[ $# -gt 0 ]]; do - case $1 in - --properties-file) - PROPERTIES_FILE="$2" - PROPERTIES="--properties-file $PROPERTIES_FILE" - shift; shift - ;; - --run) - export SESSION_TYPE="CLUSTER" - RUN_PYTHON_PHASE=1 - PYTHON_SCRIPT="$2" - shift; shift - ;; - --run-databricks) - RUN_PYTHON_DB_CONNECT_PHASE=1 - PYTHON_SCRIPT_DB_CONNECT="$2" - shift; shift - ;; - --log) - LOG_FILE=$2 - LOGGING="--files $LOG_FILE" - shift; shift - ;; - --debug-port) - DEBUG_PORT="$2" - DEBUG_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:${DEBUG_PORT}" - shift; shift - ;; - --no-suspend) - DEBUG_OPTS="${DEBUG_OPTS/suspend=y/suspend=n}" - shift - ;; - *) - POSITIONAL_ARGS+=("$1") - shift - ;; - esac -done - -set -- "${POSITIONAL_ARGS[@]}" - -# ---------- executable selection ------------------------------------------- - -if [[ $RUN_PYTHON_PHASE -eq 1 ]]; then - EXECUTABLE="$PYTHON_SCRIPT" -elif [[ $RUN_PYTHON_DB_CONNECT_PHASE -eq 1 ]]; then - EXECUTABLE="$PYTHON_SCRIPT_DB_CONNECT" -else - EXECUTABLE="--class zingg.spark.client.SparkClient $JAR_PATH" -fi - -# ---------- launch --------------------------------------------------------- - -echo "=== Zingg debug launcher ===" -echo "JAR : $JAR_PATH" -echo "SPARK : $SPARK_HOME" -echo "Debug : $DEBUG_OPTS" -echo "Waiting for debugger on port ${DEBUG_PORT} ..." -echo "" - -if [[ $RUN_PYTHON_DB_CONNECT_PHASE -eq 1 ]]; then - unset SPARK_MASTER - unset SPARK_HOME - export DATABRICKS_CONNECT=Y - python $EXECUTABLE -else - $SPARK_HOME/bin/spark-submit \ - --master $SPARK_MASTER \ - $PROPERTIES \ - --files "./log4j2.properties" \ - --conf "spark.executor.extraJavaOptions=$log4j_setting -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+HeapDumpOnOutOfMemoryError -Xloggc:/tmp/memLog.txt -XX:+UseCompressedOops" \ - --conf "spark.driver.extraJavaOptions=$log4j_setting $DEBUG_OPTS" \ - $LOGGING \ - --driver-class-path "$ZINGG_JARS" \ - $EXECUTABLE "$@" \ - --email $EMAIL --license $LICENSE -fi