spark优化参数

1、提交参数

#!/bin/bash
basePath=$(cd"$(dirname )“$(cd"$(dirname “$0”): pwd)”)": pwd)

spark-submit
–master yarn
–queue samshare
–deploy-mode client
–num-executors 100
–executor-memory 4G
–executor-cores 4
–driver-memory 2G
–driver-cores 2
–conf spark.default.parallelism=1000 \ 并行度
–conf spark.yarn.executor.memoryOverhead=8G \ executor端额外内存
–conf spark.sql.shuffle.partitions=1000 \
–conf spark.network.timeout=1200
–conf spark.python.worker.memory=64m
–conf spark.sql.catalogImplementation=hive
–conf spark.sql.crossJoin.enabled=True
–conf spark.dynamicAllocation.enabled=True
–conf spark.shuffle.service.enabled=True
–conf spark.scheduler.listenerbus.eventqueue.size=100000
–conf spark.pyspark.driver.python=python3
–conf spark.pyspark.python=python3
–conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=python3
–conf spark.sql.pivotMaxValues=500000
–conf spark.hadoop.hive.exec.dynamic.partition=True
–conf spark.hadoop.hive.exec.dynamic.partition.mode=