kind: Job

apiVersion: batch/v1

metadata:

  name: dg-job

  namespace: cti-svcs-drift-spark-etl-173543

  labels:

    app: dg-job

    batch.kubernetes.io/job-name: dg-job

    job-name: dg-job

    spark-job: Nexus

spec:

  manualSelector: false

  backoffLimit: 30

  completions: 1

  template:

    metadata:

      creationTimestamp: null

      labels:

        app: dg-job

        job-name: dg-job

        spark-job: Nexus

    spec:

      containers:

        – name: spark-driver

          imagePullPolicy: Always

          terminationMessagePolicy: File

          image: ‘docker-cto-dev-local.artifactrepository.citigroup.net/cti-svcs-drift-spark-etl-173543/dg:latest.dg-ecs’

          resources:

            limits:

              cpu: 500m

              ephemeral-storage: 10Gi

              memory: 2Gi

            requests:

              cpu: 100m

              ephemeral-storage: 100Mi

              memory: 1Gi

          env:

            – name: JAVA_TOOL_OPTIONS

              value: “-Drun_id=’77777′ -DCTIDGConfig_fileName=’/app/resources/CTIDGConfig_DEV.properties'”

          command: [“/bin/sh”,”-c”]

          args:

            – |

              /opt/spark/bin/spark-submit

              –master k8s://https://api.namanrgtd001d.ecs.dyn.nsroot.net:6443

              –deploy-mode cluster

              –name Nexus

              –conf “spark.driver.extraJavaOptions=-Drun_id=’112230′ -DCTIDGConfig_fileName=’CTIDGConfig_DEV.properties'”

              –conf spark.kubernetes.container.image=docker-cto-dev-local.artifactrepository.citigroup.net/cti-svcs-drift-spark-etl-173543/dg:latest.dg-ecs

              –conf spark.kubernetes.container.image.pullPolicy=IfNotPresent

              –conf spark.kubernetes.namespace=cti-svcs-drift-spark-etl-173543

              –conf spark.kubernetes.authenticate.driver.serviceAccountName=spark

              –conf spark.kubernetes.executor.memory=2G

              –conf spark.kubernetes.driver.memory=2G

              –conf spark.kubernetes.executor.memoryOverhead=512m

              –conf spark.kubernetes.driver.memoryOverhead=512m

              –conf spark.kubernetes.driver.request.cores=100m

              –conf spark.kubernetes.executor.request.cores=100m

              –conf spark.kubernetes.executor.deleteOnTermination=true

              –conf spark.eventLog.enabled=true

              –conf spark.eventLog.dir=/app/logs

              –conf spark.kubernetes.file.upload.path=/app/SparkTempDir

              –conf spark.kubernetes.driver.podTemplateFile=local:/app/resources/driver.yaml

              –conf spark.kubernetes.executor.podTemplateFile=local:/app/resources/executer.yaml

              –class com.citi.dg.IngestionRuleEngineProcess

              –files local:///app/resources/log4j2.properties

              ‘local:///app/jars/com.citi.dg-0.0.1-SNAPSHOT-jar-with-dependencies.jar’

              ‘/app/resources’

              ‘Nexus’

          ports:

            – name: http

              containerPort: 18080

              protocol: TCP

          readinessProbe:

            httpGet:

              path: /

              port: 18080

              scheme: HTTP

            initialDelaySeconds: 90

            timeoutSeconds: 1

            periodSeconds: 30

            successThreshold: 1

            failureThreshold: 5

          terminationMessagePath: /dev/termination-log

          livenessProbe:

            httpGet:

              path: /

              port: 18080

              scheme: HTTP

            initialDelaySeconds: 90

            timeoutSeconds: 1

            periodSeconds: 30

            successThreshold: 1

            failureThreshold: 5

      restartPolicy: Never

      terminationGracePeriodSeconds: 30

      dnsPolicy: ClusterFirst

      serviceAccountName: spark

      serviceAccount: spark

      securityContext: {}

      schedulerName: default-scheduler

  ttlSecondsAfterFinished: 600

  suspend: false

  parallelism: 1

  podReplacementpolicy: TerminatingOrFailed

  completionMode: NonIndexed