Skip to content
This repository was archived by the owner on Jul 23, 2024. It is now read-only.

Commit 52b5f9f

Browse files
author
Mariano Gonzalez
committed
PR comments
1 parent d10c5c1 commit 52b5f9f

5 files changed

Lines changed: 37 additions & 127 deletions

File tree

‎contrib/hawq-docker/Makefile‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ LOCAL :=
2929
# networks used in docker
3030
NETWORK := $(CLUSTER_ID)_hawq_network
3131
HAWQ_HOME := "/data/hawq-devel"
32+
PXF_CLASSPATH_TEMPLATE = "hdp"
3233
JAVA_TOOL_OPTIONS := -Dfile.encoding=UTF8
3334

3435
all:
@@ -39,21 +40,21 @@ all:
3940
@echo " To remove hdfs containers: make clean"
4041
@echo " To remove all containers: make distclean"
4142
@echo ""
42-
@echo " To build images locally: make build"
43+
@echo " To build images locally: make build-image"
4344
@echo " To pull latest images: make pull"
4445
@echo ""
45-
@echo " To build Hawq runtime: make hawq"
46+
@echo " To build Hawq runtime: make build-hawq"
4647
@echo " To initialize Hawq on Namenode: make init-hawq"
4748
@echo " To start Hawq on Namenode: make start-hawq"
4849
@echo " To stop Hawq on Namenode: make stop-hawq"
4950
@echo " To check Hawq status on Namenode: make status-hawq"
50-
@echo " To build PXF runtime: make pxf"
51+
@echo " To build PXF runtime: make build-pxf"
5152
@echo " To initialize PXF on Namenode/Datanodes: make init-pxf"
5253
@echo " To start PXF on Namenode/Datanodes: make start-pxf"
5354
@echo " To stop PXF on on Namenode/Datanodes: make stop-hawq"
5455
@echo " To check PXF status on Namenode/Datanodes: make status-pxf"
5556

56-
build:
57+
build-image:
5758
@make -f $(THIS_MAKEFILE_PATH) build-hawq-dev-$(OS_VERSION)
5859
@make -f $(THIS_MAKEFILE_PATH) build-hawq-test-$(OS_VERSION)
5960
@echo "Build Images Done!"
@@ -64,7 +65,10 @@ build-hawq-dev-$(OS_VERSION): $(TOP_DIR)/$(OS_VERSION)-docker/hawq-dev/Dockerfil
6465

6566
build-hawq-test-$(OS_VERSION): $(TOP_DIR)/$(OS_VERSION)-docker/hawq-test/Dockerfile
6667
@echo build hawq-test:$(OS_VERSION) image
67-
docker build -t hawq/hawq-test:$(OS_VERSION) $(TOP_DIR)/$(OS_VERSION)-docker/hawq-test/
68+
docker build \
69+
--build-arg=PXF_CLASSPATH_TEMPLATE="`cat ../../pxf/pxf-service/src/configs/templates/pxf-private-${PXF_CLASSPATH_TEMPLATE}.classpath.template`" \
70+
--build-arg=PXF_LOG4J_PROPERTIES="`cat ../../pxf/pxf-service/src/main/resources/pxf-log4j.properties`" \
71+
-t hawq/hawq-test:$(OS_VERSION) $(TOP_DIR)/$(OS_VERSION)-docker/hawq-test/
6872

6973
create-data-container:
7074
@echo create ${CLUSTER_ID}-data container
@@ -241,7 +245,8 @@ distclean:
241245
fi
242246
@echo "Distclean Done!"
243247

244-
hawq:
248+
build-hawq:
249+
@echo "Make sure you have executed make build-image"
245250
@echo "Logging into ${CLUSTER_ID}-namenode container"
246251
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
247252
docker exec \
@@ -253,7 +258,7 @@ hawq:
253258
fi
254259

255260
init-hawq:
256-
@echo "Make sure you have executed make hawq"
261+
@echo "Make sure you have executed make build-hawq"
257262
@echo "Logging into ${CLUSTER_ID}-namenode container"
258263
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
259264
docker exec \
@@ -265,6 +270,7 @@ init-hawq:
265270
fi
266271

267272
start-hawq:
273+
@echo "Make sure you have executed make init-hawq"
268274
@echo "Logging into ${CLUSTER_ID}-namenode container"
269275
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
270276
docker exec \
@@ -298,7 +304,8 @@ status-hawq:
298304
echo "${CLUSTER_ID}-namenode container does not exist!"; \
299305
fi
300306

301-
pxf:
307+
build-pxf:
308+
@echo "Make sure you have executed make build-image"
302309
@make -f $(THIS_MAKEFILE_PATH) pxf-namenode
303310
@i=1; \
304311
while [ $$i -le $(NDATANODES) ] ; do \
@@ -312,6 +319,7 @@ pxf-namenode:
312319
docker exec \
313320
-e "HAWQ_HOME=$(HAWQ_HOME)" \
314321
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
322+
-e "PXF_CLASSPATH_TEMPLATE=$(PXF_CLASSPATH_TEMPLATE)" \
315323
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --build"; \
316324
else \
317325
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
@@ -329,7 +337,7 @@ pxf-datanode:
329337
fi
330338

331339
init-pxf:
332-
@echo "Make sure you have executed make pxf"
340+
@echo "Make sure you have executed make build-pxf"
333341
@make -f $(THIS_MAKEFILE_PATH) init-pxf-namenode
334342
@i=1; \
335343
while [ $$i -le $(NDATANODES) ] ; do \
@@ -360,6 +368,7 @@ init-pxf-datanode:
360368
fi
361369

362370
start-pxf:
371+
@echo "Make sure you have executed make init-pxf"
363372
@make -f $(THIS_MAKEFILE_PATH) start-pxf-namenode
364373
@i=1; \
365374
while [ $$i -le $(NDATANODES) ] ; do \

‎contrib/hawq-docker/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -121,14 +121,14 @@ make pull
121121
To stop all containers: make stop
122122
To remove hdfs containers: make clean
123123
To remove all containers: make distclean
124-
To build images locally: make build
124+
To build images locally: make build-image
125125
To pull latest images: make pull
126-
To build Hawq runtime: make hawq
126+
To build Hawq runtime: make build-hawq
127127
To initialize Hawq on Namenode: make init-hawq
128128
To start Hawq on Namenode: make start-hawq
129129
To stop Hawq on Namenode: make stop-hawq
130130
To check Hawq status on Namenode: make status-hawq
131-
To build PXF runtime: make pxf
131+
To build PXF runtime: make build-pxf
132132
To initialize PXF on Namenode/Datanodes: make init-pxf
133133
To start PXF on Namenode/Datanodes: make start-pxf
134134
To stop PXF on on Namenode/Datanodes: make stop-hawq

‎contrib/hawq-docker/centos7-docker/hawq-test/Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,21 @@ MAINTAINER Richard Guo <riguo@pivotal.io>
2121

2222
USER root
2323

24+
ARG PXF_CLASSPATH_TEMPLATE
25+
ARG PXF_LOG4J_PROPERTIES
26+
2427
## install HDP 2.5.0
2528
RUN curl -L "http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.5.0.0/hdp.repo" -o /etc/yum.repos.d/hdp.repo && \
2629
yum install -y hadoop hadoop-hdfs hadoop-libhdfs hadoop-yarn hadoop-mapreduce hadoop-client hdp-select && \
2730
yum clean all
2831

2932
RUN ln -s /usr/hdp/current/hadoop-hdfs-namenode/../hadoop/sbin/hadoop-daemon.sh /usr/bin/hadoop-daemon.sh
3033

34+
RUN touch /tmp/pxf-private.classpath && \
35+
touch /tmp/log4j.properties && \
36+
echo "$PXF_CLASSPATH_TEMPLATE" > /tmp/pxf-private.classpath && \
37+
echo "$PXF_LOG4J_PROPERTIES" > /tmp/pxf-log4j.properties
38+
3139
COPY conf/* /etc/hadoop/conf/
3240

3341
COPY entrypoint.sh /usr/bin/entrypoint.sh

‎contrib/hawq-docker/centos7-docker/hawq-test/service-hawq.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ start() {
6565
source ${HAWQ_HOME}/greenplum_path.sh
6666
echo "Starting HAWQ Cluster"
6767
hawq start cluster -a
68-
echo "Starting HAWQ Cluster"
68+
echo "Starting HAWQ Cluster Done!"
6969
}
7070

7171
stop() {

‎contrib/hawq-docker/centos7-docker/hawq-test/service-pxf.sh‎

Lines changed: 7 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -24,126 +24,19 @@ build() {
2424
sudo echo "source ${GPHOME}/greenplum_path.sh" >> /home/gpadmin/.bashrc
2525

2626
cd /data/hawq/pxf
27-
make -j8
27+
#make -j16
28+
make
2829
make install
2930

3031
sudo sed 's|-pxf|-gpadmin|g' -i ${PXF_HOME}/conf/pxf-env.sh
3132

33+
rm -rf ${PXF_HOME}/conf/pxf-private.classpath
3234
rm -rf ${PXF_HOME}/conf/pxf-log4j.properties
33-
cat <<EOF >>${PXF_HOME}/conf/pxf-log4j.properties
34-
# Licensed to the Apache Software Foundation (ASF) under one
35-
# or more contributor license agreements. See the NOTICE file
36-
# distributed with this work for additional information
37-
# regarding copyright ownership. The ASF licenses this file
38-
# to you under the Apache License, Version 2.0 (the
39-
# "License"); you may not use this file except in compliance
40-
# with the License. You may obtain a copy of the License at
41-
#
42-
# http://www.apache.org/licenses/LICENSE-2.0
43-
#
44-
# Unless required by applicable law or agreed to in writing,
45-
# software distributed under the License is distributed on an
46-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
47-
# KIND, either express or implied. See the License for the
48-
# specific language governing permissions and limitations
49-
# under the License.
50-
log4j.appender.ROLLINGFILE.File=${PXF_HOME}/pxf-service/logs/pxf-service.log
51-
log4j.appender.ROLLINGFILE.MaxFileSize=10MB
52-
log4j.appender.ROLLINGFILE.MaxBackupIndex=10
53-
log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
54-
log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSSS} %p %t %c - %m%n
55-
EOF
5635

57-
rm -rf ${PXF_HOME}/conf/pxf-private.classpath
58-
cat <<EOF >>${PXF_HOME}/conf/pxf-private.classpath
59-
# Licensed to the Apache Software Foundation (ASF) under one or more
60-
# contributor license agreements. See the NOTICE file distributed with
61-
# this work for additional information regarding copyright ownership.
62-
# The ASF licenses this file to You under the Apache License, Version 2.0
63-
# (the "License"); you may not use this file except in compliance with
64-
# the License. You may obtain a copy of the License at
65-
#
66-
# http://www.apache.org/licenses/LICENSE-2.0
67-
#
68-
# Unless required by applicable law or agreed to in writing, software
69-
# distributed under the License is distributed on an "AS IS" BASIS,
70-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
71-
# See the License for the specific language governing permissions and
72-
# limitations under the License.
73-
74-
##################################################################
75-
# This file contains the internal classpaths required to run PXF.
76-
# Edit to set the base paths according to your specific package layout
77-
# Adding new resources should be done using pxf-public.classpath file.
78-
##################################################################
79-
80-
# PXF Configuration
81-
/data/hawq-devel/pxf/conf
82-
83-
# Hadoop Configuration
84-
/usr/hdp/2.5.0.0-1245/hadoop/etc/hadoop
85-
86-
# Hive Configuration
87-
# hive/conf
88-
89-
# Hbase Configuration
90-
# base/conf
91-
92-
# PXF Libraries
93-
/data/hawq-devel/pxf/lib/pxf-hbase-*[0-9].jar
94-
/data/hawq-devel/pxf/lib/pxf-hdfs-*[0-9].jar
95-
/data/hawq-devel/pxf/lib/pxf-hive-*[0-9].jar
96-
/data/hawq-devel/pxf/lib/pxf-json-*[0-9].jar
97-
/data/hawq-devel/pxf/lib/pxf-jdbc-*[0-9].jar
98-
/data/hawq-devel/pxf/lib/pxf-ignite-*[0-9].jar
99-
100-
# Hadoop Libraries
101-
/usr/hdp/2.5.0.0-1245/hadoop/client/hadoop-hdfs-*[0-9].jar
102-
/usr/hdp/2.5.0.0-1245/hadoop/client/hadoop-mapreduce-client-core-*[0-9].jar
103-
/usr/hdp/2.5.0.0-1245/hadoop/client/hadoop-auth-*[0-9].jar
104-
/usr/hdp/2.5.0.0-1245/hadoop/client/hadoop-common-*[0-9].jar
105-
/usr/hdp/2.5.0.0-1245/hadoop/lib/asm-*[0-9].jar
106-
/usr/hdp/2.5.0.0-1245/hadoop/client/avro-*[0-9].jar
107-
/usr/hdp/2.5.0.0-1245/hadoop/client/commons-cli-*[0-9].jar
108-
/usr/hdp/2.5.0.0-1245/hadoop/client/commons-codec-*[0-9].jar
109-
/usr/hdp/2.5.0.0-1245/hadoop/client/commons-collections-*[0-9].jar
110-
/usr/hdp/2.5.0.0-1245/hadoop/client/commons-configuration-*[0-9].jar
111-
/usr/hdp/2.5.0.0-1245/hadoop/client/commons-io-*[0-9].jar
112-
/usr/hdp/2.5.0.0-1245/hadoop/client/commons-lang-*[0-9].jar
113-
/usr/hdp/2.5.0.0-1245/hadoop/client/commons-logging-*[0-9].jar
114-
/usr/hdp/2.5.0.0-1245/hadoop/client/commons-compress-*[0-9].jar
115-
/usr/hdp/2.5.0.0-1245/hadoop/client/guava-*[0-9].jar
116-
/usr/hdp/2.5.0.0-1245/hadoop/client/htrace-core*.jar
117-
/usr/hdp/2.5.0.0-1245/hadoop/client/jetty-*.jar
118-
/usr/hdp/2.5.0.0-1245/hadoop/client/jackson-core-asl-*[0-9].jar
119-
/usr/hdp/2.5.0.0-1245/hadoop/client/jackson-mapper-asl-*[0-9].jar
120-
/usr/hdp/2.5.0.0-1245/hadoop/lib/jersey-core-*[0-9].jar
121-
/usr/hdp/2.5.0.0-1245/hadoop/lib/jersey-server-*[0-9].jar
122-
/usr/hdp/2.5.0.0-1245/hadoop/client/log4j-*[0-9].jar
123-
/usr/hdp/2.5.0.0-1245/hadoop/client/protobuf-java-*[0-9].jar
124-
/usr/hdp/2.5.0.0-1245/hadoop/client/slf4j-api-*[0-9].jar
125-
/usr/hdp/2.5.0.0-1245/hadoop/client/gson-*[0-9].jar
126-
127-
# Hive Libraries
128-
# hive/lib/antlr-runtime*.jar
129-
# hive/lib/datanucleus-api-jdo*.jar
130-
# hive/lib/datanucleus-core*.jar
131-
# hive/lib/hive-exec*.jar
132-
# hive/lib/hive-metastore*.jar
133-
# hive/lib/jdo-api*.jar
134-
# hive/lib/libfb303*.jar
135-
# when running on OSx, 1.0.5 or higher version is required
136-
# hive/lib/snappy-java*.jar
137-
138-
# HBase Libraries
139-
# hbase/lib/hbase-client*.jar
140-
# hbase/lib/hbase-common*.jar
141-
# hbase/lib/hbase-protocol*.jar
142-
# hbase/lib/htrace-core*.jar
143-
# hbase/lib/netty*.jar
144-
# hbase/lib/zookeeper*.jar
145-
# hbase/lib/metrics-core*.jar
146-
EOF
36+
sudo cp /tmp/pxf-private.classpath ${PXF_HOME}/conf/pxf-private.classpath
37+
sudo cp /tmp/pxf-log4j.properties ${PXF_HOME}/conf/pxf-log4j.properties
38+
sudo sed 's|pxf\.log\.dir|PXF_HOME|g' -i ${PXF_HOME}/conf/pxf-log4j.properties
39+
14740
echo "Make PXF Done!"
14841
}
14942

0 commit comments

Comments
 (0)