用phoenix写hbase scala版

用phoenix写hbase scala版

开发环境：
IDEA
HBase版本：2.1.0-cdh6.1.0
spark版本：spark-core_2.11
由于使用的HBase版本过高，在Phoenix官网找不到对应的Phoenix版本，所以只能使用目前发布的最新版5.0.0-HBase-2.0

pom文件

<?xml version="1.0" encoding="UTF-8"?>

<groupId>com.</groupId>
<artifactId>Unicom-test</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
    <repository>
        <id>cloudera-releases</id>
        <url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
        <releases>
            <enabled>true</enabled>
        </releases>
        <snapshots>
            <enabled>false</enabled>
        </snapshots>
    </repository>
</repositories>
<dependencies>
    <dependency>
        <!--spark-core的依赖-->
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_2.11</artifactId>
        <version>2.2.0</version>
    </dependency>
    <!--spark-sql的依赖-->
    <!--<dependency>-->
    <!--<groupId>org.apache.spark</groupId>-->
    <!--<artifactId>spark-sql_2.11</artifactId>-->
    <!--<version>2.2.0</version>-->
    <!--</dependency>-->
    <!--spark-streaming的依赖-->
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming_2.11</artifactId>
        <version>2.2.0</version>
    </dependency>
    <!--spark-streaming对接kafka的依赖-->
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
        <version>2.2.0</version>
    </dependency>
    <!--kafka客户端-->
    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka-clients</artifactId>
        <version>0.10.2.1</version>
    </dependency>
    <!--解析json-->
    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>1.2.47</version>
    </dependency>
    <!--加载配置文件-->
    <dependency>
        <groupId>com.typesafe</groupId>
        <artifactId>config</artifactId>
        <version>1.2.1</version>
    </dependency>
    <!--log4j-->
    <dependency>
        <groupId>log4j</groupId>
        <artifactId>log4j</artifactId>
        <version>1.2.17</version>
    </dependency>
    <!--phoenix-hbase的依赖-->
    <!-- https://mvnrepository.com/artifact/org.apache.phoenix/phoenix -->
    <dependency>
        <groupId>org.apache.phoenix</groupId>
        <artifactId>phoenix-spark</artifactId>
        <version>5.0.0-HBase-2.0</version>
        <scope>provided</scope>
        <type>pom</type>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.phoenix/phoenix-core -->
    <dependency>
        <groupId>org.apache.phoenix</groupId>
        <artifactId>phoenix-core</artifactId>
        <version>5.0.0-HBase-2.0</version>
    </dependency>
</dependencies>
<!--自己的打包方式-->
<build>
    <sourceDirectory>src/main/scala</sourceDirectory>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.0.0</version>
            <executions>
                <execution>
                    <phase>package</phase>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                    <configuration>
                        <transformers>
                            <transformer
                                    implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                <mainClass>kafka2Spark2Hbase.test.Kafka2HbaseByPhoenix</mainClass>
                            </transformer>
                        </transformers>
                    </configuration>
                </execution>
            </executions>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.1</version>
        </plugin>
        <plugin>
            <groupId>net.alchim31.maven</groupId>
            <artifactId>scala-maven-plugin</artifactId>
            <version>3.2.2</version>
            <executions>
                <execution>
                    <goals>
                        <goal>compile</goal>
                        <goal>testCompile</goal>
                    </goals>
                    <configuration>
                        <args>
                            <arg>-dependencyfile</arg>
                            <arg>${project.build.directory}/.scala_dependencies</arg>
                        </args>
                    </configuration>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

测试代码：

import java.sql.{Connection, DriverManager}
import java.util.Properties

import com.typesafe.config.{Config, ConfigFactory}

object ConfigHelper {
 /**
    * 加载application.conf文件
    */
  private lazy val application: Config = ConfigFactory.load()
  /**
    * phoenix的参数信息
    */
  val PHOENIX_DRIVER: String = application.getString("phoenix.driver")
  val PHOENIX_URL: String = application.getString("phoenix.url")
    val PROPS = new Properties()
  PROPS.setProperty("driver", PHOENIX_DRIVER)
    // 因为开启了 namespace mapping，所以需要设置改参数为 true
  PROPS.setProperty("phoenix.schema.isNamespaceMappingEnabled", "true")
  val connection: Connection = DriverManager.getConnection(PHOENIX_URL, PROPS)
  }

object PhoenixTest {
def main(args: Array[String]): Unit = {
  val connection: Connection = DriverManager.getConnection(PHOENIX_URL, PROPS)
  val createSql = "CREATE TABLE IF NOT EXISTS USER2 (
  "serial_number" VARCHAR  NOT NULL,
  "user_id" VARCHAR NOT NULL,
  "province_code" VARCHAR,
  "eparchy_code" VARCHAR ,
  "eparchy_name" VARCHAR,
  "product_id" VARCHAR
   CONSTRAINT "user" PRIMARY KEY ("serial_number","user_id"));"
  val preparedStatement: PreparedStatement = connection.prepareStatement(createSql)
    connection.setAutoCommit(false)
    preparedStatement.execute()
    connection.commit()
    preparedStatement.close()
    connection.close()
  }
}