Skip to content

Commit 6a4ad18

Browse files
inchiosainchiosa
committed
Add datasets and redirection
1 parent e5f086d commit 6a4ad18

2 files changed

Lines changed: 19 additions & 49 deletions

File tree

‎samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda8.ipynb‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -29,63 +29,49 @@
2929
{
3030
"cell_type": "code",
3131
"source": "# For informational purposes,\r\n# print the hostname of the container\r\n# where the Spark driver is running\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n \"hostname\",\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
32-
"metadata": {
33-
"language": "python"
34-
},
32+
"metadata": {},
3533
"outputs": [],
3634
"execution_count": 3
3735
},
3836
{
3937
"cell_type": "code",
4038
"source": "# Install NVIDIA GPU libraries and TensorFlow for GPU\r\n# in the container where the Spark driver is running\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\necho $CUDA_VERSION\r\nexport CUDA_PKG_VERSION=\"8-0=$CUDA_VERSION-1\"\r\necho $CUDA_PKG_VERSION\r\n\r\nexport PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}\r\nexport LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64\r\n\r\n# nvidia-container-runtime\r\nexport NVIDIA_VISIBLE_DEVICES=all\r\nexport NVIDIA_DRIVER_CAPABILITIES=\"compute,utility\"\r\nexport NVIDIA_REQUIRE_CUDA=\"cuda>=8.0\"\r\n\r\napt-get update && apt-get install -y --no-install-recommends ca-certificates apt-transport-https gnupg-curl && \\\\\r\n rm -rf /var/lib/apt/lists/* && \\\\\r\n NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \\\\\r\n NVIDIA_GPGKEY_FPR=ae09fe4bbd223a84b2ccfce3f60f4b3d7fa2af80 && \\\\\r\n apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub && \\\\\r\n apt-key adv --export --no-emit-version -a $NVIDIA_GPGKEY_FPR | tail -n +5 > cudasign.pub && \\\\\r\n echo \"$NVIDIA_GPGKEY_SUM cudasign.pub\" | sha256sum -c --strict - && rm cudasign.pub && \\\\\r\n echo \"deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /\" > /etc/apt/sources.list.d/cuda.list\r\n\r\napt-get update && apt-get install -y --no-install-recommends \\\\\r\n cuda-nvrtc-$CUDA_PKG_VERSION \\\\\r\n cuda-nvgraph-$CUDA_PKG_VERSION \\\\\r\n cuda-cusolver-$CUDA_PKG_VERSION \\\\\r\n cuda-cublas-8-0=8.0.61.2-1 \\\\\r\n cuda-cufft-$CUDA_PKG_VERSION \\\\\r\n cuda-curand-$CUDA_PKG_VERSION \\\\\r\n cuda-cusparse-$CUDA_PKG_VERSION \\\\\r\n cuda-npp-$CUDA_PKG_VERSION \\\\\r\n cuda-cudart-$CUDA_PKG_VERSION && \\\\\r\n ln -s cuda-8.0 /usr/local/cuda && \\\\\r\n rm -rf /var/lib/apt/lists/*\r\n\r\n# Install tensorflow\r\npip3 install tensorflow-gpu==1.4.0\r\n\r\n# add cudnn 6\r\necho \"deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /\" > /etc/apt/sources.list.d/nvidia-ml.list\r\n\r\nexport CUDNN_VERSION=6.0.21\r\n#LABEL com.nvidia.cudnn.version=\"${CUDNN_VERSION}\"\r\n\r\napt-get update && apt-get install -y --no-install-recommends \\\\\r\n libcudnn6=$CUDNN_VERSION-1+cuda8.0 && \\\\\r\n rm -rf /var/lib/apt/lists/*\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
41-
"metadata": {
42-
"language": "python"
43-
},
39+
"metadata": {},
4440
"outputs": [],
4541
"execution_count": 4
4642
},
4743
{
4844
"cell_type": "code",
4945
"source": "# List CPU and GPU devices\r\nfrom tensorflow.python.client import device_lib\r\n\r\ndevice_lib.list_local_devices()",
50-
"metadata": {
51-
"language": "python"
52-
},
46+
"metadata": {},
5347
"outputs": [],
5448
"execution_count": 5
5549
},
5650
{
5751
"cell_type": "code",
5852
"source": "# Fit and evaluate TensorFlow model on MNIST data\r\nimport tensorflow as tf\r\nmnist = tf.keras.datasets.mnist\r\n\r\n(x_train, y_train),(x_test, y_test) = mnist.load_data()\r\nx_train, x_test = x_train / 255.0, x_test / 255.0\r\n\r\nmodel = tf.keras.models.Sequential([\r\n tf.keras.layers.Flatten(input_shape=(28, 28)), # input_shape needed for older tensorflow\r\n tf.keras.layers.Dense(512, activation=tf.nn.relu),\r\n tf.keras.layers.Dropout(0.2),\r\n tf.keras.layers.Dense(10, activation=tf.nn.softmax)\r\n])\r\nmodel.compile(optimizer='adam',\r\n loss='sparse_categorical_crossentropy',\r\n metrics=['accuracy'])\r\n\r\nmodel.fit(x_train, y_train, epochs=5)\r\nprint(\"\\n\")\r\nmetrics = model.evaluate(x_test, y_test)\r\nprint(\"\\n\")\r\nprint(metrics)",
59-
"metadata": {
60-
"language": "python"
61-
},
53+
"metadata": {},
6254
"outputs": [],
6355
"execution_count": 11
6456
},
6557
{
6658
"cell_type": "code",
6759
"source": "# Check available disk space\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\ndf -h\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
68-
"metadata": {
69-
"language": "python"
70-
},
60+
"metadata": {},
7161
"outputs": [],
7262
"execution_count": 12
7363
},
7464
{
7565
"cell_type": "code",
76-
"source": "# Download code for the CIFAR 10 benchmark\r\nimport subprocess\r\nimport os\r\n\r\nif os.path.isdir(\"/tmp/models\"):\r\n print(\"CIFAR 10 repo already cloned\")\r\nelse:\r\n stdout = subprocess.check_output(\r\n'''\r\napt-get update && apt-get install -y git\r\ncd /tmp\r\ngit clone https://github.com/tensorflow/models.git\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\n print(stdout)",
77-
"metadata": {
78-
"language": "python"
79-
},
66+
"source": "# Download code for the CIFAR 10 benchmark\r\nimport subprocess\r\nimport os\r\n\r\nif os.path.isdir(\"/tmp/models\"):\r\n print(\"CIFAR 10 repo already cloned\")\r\nelse:\r\n stdout = subprocess.check_output(\r\n'''\r\napt-get update && apt-get install -y git\r\npip3 install --upgrade pip setuptools\r\npip3 install tensorflow-datasets\r\ncd /tmp\r\ngit clone https://github.com/tensorflow/models.git\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\n print(stdout)",
67+
"metadata": {},
8068
"outputs": [],
8169
"execution_count": 13
8270
},
8371
{
8472
"cell_type": "code",
85-
"source": "# Run the CIFAR 10 benchmark\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\npython3 /tmp/models/tutorials/image/cifar10/cifar10_train.py --max_steps 100\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
86-
"metadata": {
87-
"language": "python"
88-
},
73+
"source": "# Run the CIFAR 10 benchmark\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\npython3 /tmp/models/tutorials/image/cifar10/cifar10_train.py --max_steps 100 2>&1\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
74+
"metadata": {},
8975
"outputs": [],
9076
"execution_count": 14
9177
}

‎samples/features/sql-big-data-cluster/machine-learning/spark/tensorflow/tf-cuda9.ipynb‎

Lines changed: 10 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -29,72 +29,56 @@
2929
{
3030
"cell_type": "code",
3131
"source": "# For informational purposes,\r\n# print the hostname of the container\r\n# where the Spark driver is running\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n \"hostname\",\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
32-
"metadata": {
33-
"language": "python"
34-
},
32+
"metadata": {},
3533
"outputs": [],
3634
"execution_count": 3
3735
},
3836
{
3937
"cell_type": "code",
4038
"source": "# Check that the CUDA_VERSION environment variable is set.\r\n# Its precise value does not matter: one can install CUDA 9 even if the \r\n# CUDA_VERSION environment variable is set to 8.0.61.\r\nimport os\r\nprint(os.environ['CUDA_VERSION'])",
41-
"metadata": {
42-
"language": "python"
43-
},
39+
"metadata": {},
4440
"outputs": [],
4541
"execution_count": 4
4642
},
4743
{
4844
"cell_type": "code",
4945
"source": "# Install NVIDIA GPU libraries and TensorFlow for GPU\r\n# in the container where the Spark driver is running\r\n# per https://www.tensorflow.org/install/gpu#ubuntu_1604_cuda_90_for_tensorflow_1130\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\n# Add NVIDIA package repository\r\napt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub\r\n\r\nwget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_9.1.85-1_amd64.deb\r\n\r\nchown _apt cuda-repo-ubuntu1604_9.1.85-1_amd64.deb\r\nchmod u+rwx cuda-repo-ubuntu1604_9.1.85-1_amd64.deb\r\n\r\napt install ./cuda-repo-ubuntu1604_9.1.85-1_amd64.deb\r\n\r\nwget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb\r\n\r\nchown _apt nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb\r\nchmod u+rwx nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb\r\n\r\napt install ./nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb\r\n\r\napt update\r\n\r\n# Install CUDA and tools. Include optional NCCL 2.x\r\napt install -y cuda9.0 cuda-cublas-9-0 cuda-cufft-9-0 cuda-curand-9-0 \\\\\r\n cuda-cusolver-9-0 cuda-cusparse-9-0 libcudnn7=7.2.1.38-1+cuda9.0 \\\\\r\n libnccl2=2.2.13-1+cuda9.0 cuda-command-line-tools-9-0\r\n\r\n# Optional: Install the TensorRT runtime (must be after CUDA install)\r\napt update\r\napt install libnvinfer4=4.1.2-1+cuda9.0\r\n\r\npip3 install tensorflow-gpu==1.12.0\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
50-
"metadata": {
51-
"language": "python"
52-
},
46+
"metadata": {},
5347
"outputs": [],
5448
"execution_count": 5
5549
},
5650
{
5751
"cell_type": "code",
5852
"source": "# List CPU and GPU devices\r\nfrom tensorflow.python.client import device_lib\r\n\r\ndevice_lib.list_local_devices()",
59-
"metadata": {
60-
"language": "python"
61-
},
53+
"metadata": {},
6254
"outputs": [],
6355
"execution_count": 6
6456
},
6557
{
6658
"cell_type": "code",
6759
"source": "# Fit and evaluate TensorFlow model on MNIST data\r\nimport tensorflow as tf\r\nmnist = tf.keras.datasets.mnist\r\n\r\n(x_train, y_train),(x_test, y_test) = mnist.load_data()\r\nx_train, x_test = x_train / 255.0, x_test / 255.0\r\n\r\nmodel = tf.keras.models.Sequential([\r\n tf.keras.layers.Flatten(),\r\n tf.keras.layers.Dense(512, activation=tf.nn.relu),\r\n tf.keras.layers.Dropout(0.2),\r\n tf.keras.layers.Dense(10, activation=tf.nn.softmax)\r\n])\r\nmodel.compile(optimizer='adam',\r\n loss='sparse_categorical_crossentropy',\r\n metrics=['accuracy'])\r\n\r\nmodel.fit(x_train, y_train, epochs=5)\r\nprint(\"\\n\")\r\nmetrics = model.evaluate(x_test, y_test)\r\nprint(\"\\n\")\r\nprint(metrics)",
68-
"metadata": {
69-
"language": "python"
70-
},
60+
"metadata": {},
7161
"outputs": [],
7262
"execution_count": 7
7363
},
7464
{
7565
"cell_type": "code",
7666
"source": "# Check available disk space\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\ndf -h\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
77-
"metadata": {
78-
"language": "python"
79-
},
67+
"metadata": {},
8068
"outputs": [],
8169
"execution_count": 8
8270
},
8371
{
8472
"cell_type": "code",
85-
"source": "# Download code for the CIFAR 10 benchmark\r\nimport subprocess\r\nimport os\r\n\r\nif os.path.isdir(\"/tmp/models\"):\r\n print(\"CIFAR 10 repo already cloned\")\r\nelse:\r\n stdout = subprocess.check_output(\r\n'''\r\napt-get update && apt-get install -y git\r\ncd /tmp\r\ngit clone https://github.com/tensorflow/models.git\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\n print(stdout)",
86-
"metadata": {
87-
"language": "python"
88-
},
73+
"source": "# Download code for the CIFAR 10 benchmark\r\nimport subprocess\r\nimport os\r\n\r\nif os.path.isdir(\"/tmp/models\"):\r\n print(\"CIFAR 10 repo already cloned\")\r\nelse:\r\n stdout = subprocess.check_output(\r\n'''\r\napt-get update && apt-get install -y git\r\npip3 install --upgrade pip setuptools\r\npip3 install tensorflow-datasets\r\ncd /tmp\r\ngit clone https://github.com/tensorflow/models.git\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\n print(stdout)",
74+
"metadata": {},
8975
"outputs": [],
9076
"execution_count": 9
9177
},
9278
{
9379
"cell_type": "code",
94-
"source": "# Run the CIFAR 10 benchmark\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\npython3 /tmp/models/tutorials/image/cifar10/cifar10_train.py --max_steps 100\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
95-
"metadata": {
96-
"language": "python"
97-
},
80+
"source": "# Run the CIFAR 10 benchmark\r\nimport subprocess\r\n\r\nstdout = subprocess.check_output(\r\n'''\r\npython3 /tmp/models/tutorials/image/cifar10/cifar10_train.py --max_steps 100 2>&1\r\n''',\r\n stderr=subprocess.STDOUT,\r\n shell=True).decode(\"utf-8\")\r\nprint(stdout)",
81+
"metadata": {},
9882
"outputs": [],
9983
"execution_count": 10
10084
}

0 commit comments

Comments
 (0)