jydoop/Makefile

HBASE_PATH ?= /usr/lib/hbase
SPACE := $(NULL) $(NULL)
HBASE_CP = $(subst $(SPACE),:,$(wildcard $(HBASE_PATH)/*.jar) $(wildcard $(HBASE_PATH)/lib/*.jar))

#javac -classpath   HBaseDriver.java  -d out  -Xlint:deprecation  && jar -cvf taras.jar -C out/ . 
export HADOOP_USER_CLASSPATH_FIRST="true"
# this will need to change once more jars are added
export HADOOP_CLASSPATH=jython-standalone-2.7-b1.jar:akela-0.5-SNAPSHOT.jar:jackson-core-2.1.4.jar
CP=$(HADOOP_CLASSPATH):$(HBASE_CP)
comma:=,
JAVA_SOURCE=$(addprefix org/mozilla/jydoop/,PythonWrapper.java PythonValue.java PythonKey.java HBaseDriver.java JacksonWrapper.java)
TASK=HBaseDriver
ARGS=input output
SCRIPT=$(error Must specify SCRIPT=)
TEST_PY=test.py
all: driver.jar

check: driver.jar
	java -cp driver.jar:$(CP) org.python.util.jython $(TEST_PY)

run: driver.jar
	java -cp driver.jar:$(CP) org.mozilla.jydoop.$(TASK)

hadoop: driver.jar
	time hadoop jar $< org.mozilla.jydoop.$(TASK) -libjars $(subst :,$(comma),$(HADOOP_CLASSPATH)) $(ARGS)

out/pylib:
	mkdir -p out
	ln -s ../pylib out/pylib

out/scripts:
	mkdir -p out
	ln -s ../scripts out/scripts

driver.jar: out/scripts out/pylib $(wildcard scripts/*.py) $(wildcard pylib/*.py) $(JAVA_SOURCE)
	javac -Xlint:deprecation -d out  -cp $(CP) $(JAVA_SOURCE)
	jar -cvf $@ -C out .

%.class: ../%.java

download:
	wget -c http://repo1.maven.org/maven2/org/python/jython-standalone/2.7-b1/jython-standalone-2.7-b1.jar -O jython-standalone-2.7-b1.jar
	wget -c http://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.1.4/jackson-core-2.1.4.jar -O jackson-core-2.1.4.jar
	wget -c http://people.mozilla.org/~bsmedberg/akela-0.5-SNAPSHOT.jar -O akela-0.5-SNAPSHOT.jar
Make the path of hbase configurable, and make it easier to build. 2013-03-21 12:50:31 +04:00			`HBASE_PATH ?= /usr/lib/hbase`
			`SPACE := $(NULL) $(NULL)`
			`HBASE_CP = $(subst $(SPACE),:,$(wildcard $(HBASE_PATH)/.jar) $(wildcard $(HBASE_PATH)/lib/.jar))`

some cleanup 2013-03-13 23:12:53 +04:00			`#javac -classpath HBaseDriver.java -d out -Xlint:deprecation && jar -cvf taras.jar -C out/ .`
first successful map/reduce with json 2013-03-07 09:12:37 +04:00			`export HADOOP_USER_CLASSPATH_FIRST="true"`
got wordcount to run partially in python 2013-03-07 00:02:18 +04:00			`# this will need to change once more jars are added`
Use Jackson instead of Jyson for speed. See also TODO for future speed improvements in JacksonWrapper.loads. 2013-04-06 20:35:28 +04:00			`export HADOOP_CLASSPATH=jython-standalone-2.7-b1.jar:akela-0.5-SNAPSHOT.jar:jackson-core-2.1.4.jar`
Make the path of hbase configurable, and make it easier to build. 2013-03-21 12:50:31 +04:00			`CP=$(HADOOP_CLASSPATH):$(HBASE_CP)`
first successful map/reduce with json 2013-03-07 09:12:37 +04:00			`comma:=,`
Separate the type and value classes so that values can contain lists and dicts (and are not sortable) while keys cannot contain those and are sortable. 2013-04-19 20:35:57 +04:00			`JAVA_SOURCE=$(addprefix org/mozilla/jydoop/,PythonWrapper.java PythonValue.java PythonKey.java HBaseDriver.java JacksonWrapper.java)`
some cleanup 2013-03-13 23:12:53 +04:00			`TASK=HBaseDriver`
got jython running against hbase too 2013-03-13 10:33:49 +04:00			`ARGS=input output`
Implement tests for TypeWritable and its implementation of RawComparator. 2013-04-04 02:18:55 +04:00			`SCRIPT=$(error Must specify SCRIPT=)`
added a way to pull resources out of driver.jar 2013-04-19 03:28:35 +04:00			`TEST_PY=test.py`
some cleanup 2013-03-13 23:12:53 +04:00			`all: driver.jar`
got wordcount to run partially in python 2013-03-07 00:02:18 +04:00
Implement tests for TypeWritable and its implementation of RawComparator. 2013-04-04 02:18:55 +04:00			`check: driver.jar`
added a way to pull resources out of driver.jar 2013-04-19 03:28:35 +04:00			`java -cp driver.jar:$(CP) org.python.util.jython $(TEST_PY)`
Implement tests for TypeWritable and its implementation of RawComparator. 2013-04-04 02:18:55 +04:00
some cleanup 2013-03-13 23:12:53 +04:00			`run: driver.jar`
Rename pydoop->jydoop because there is already a pretty well-known project called pydoop. 2013-04-09 03:07:14 +04:00			`java -cp driver.jar:$(CP) org.mozilla.jydoop.$(TASK)`
got wordcount to run partially in python 2013-03-07 00:02:18 +04:00
some cleanup 2013-03-13 23:12:53 +04:00			`hadoop: driver.jar`
Rename pydoop->jydoop because there is already a pretty well-known project called pydoop. 2013-04-09 03:07:14 +04:00			`time hadoop jar $< org.mozilla.jydoop.$(TASK) -libjars $(subst :,$(comma),$(HADOOP_CLASSPATH)) $(ARGS)`
got wordcount to run partially in python 2013-03-07 00:02:18 +04:00
Move common modules to jydoop directory 2013-04-20 23:07:01 +04:00			`out/pylib:`
			`mkdir -p out`
			`ln -s ../pylib out/pylib`

Ship all the scripts in driver.jar and select which one to run at runtime (preparation for a standard library of pydoop python scripts). 2013-04-04 20:27:20 +04:00			`out/scripts:`
			`mkdir -p out`
			`ln -s ../scripts out/scripts`

Move common modules to jydoop directory 2013-04-20 23:07:01 +04:00			`driver.jar: out/scripts out/pylib $(wildcard scripts/.py) $(wildcard pylib/.py) $(JAVA_SOURCE)`
Move the Java into the correct directory structure expected by a java compiler, and give it a more correct package name. Add a Writeable class which is used to efficiently serialize just the data we might actually want in a python key/value, so that we can now emit tuples and integers. 2013-03-22 06:27:37 +04:00			`javac -Xlint:deprecation -d out -cp $(CP) $(JAVA_SOURCE)`
got wordcount to run partially in python 2013-03-07 00:02:18 +04:00			`jar -cvf $@ -C out .`

			`%.class: ../%.java`
Some more docs 2013-03-18 19:45:52 +04:00
			`download:`
Use wget -c per taras 2013-03-21 13:38:37 +04:00			`wget -c http://repo1.maven.org/maven2/org/python/jython-standalone/2.7-b1/jython-standalone-2.7-b1.jar -O jython-standalone-2.7-b1.jar`
Use Jackson instead of Jyson for speed. See also TODO for future speed improvements in JacksonWrapper.loads. 2013-04-06 20:35:28 +04:00			`wget -c http://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.1.4/jackson-core-2.1.4.jar -O jackson-core-2.1.4.jar`
Use wget -c per taras 2013-03-21 13:38:37 +04:00			`wget -c http://people.mozilla.org/~bsmedberg/akela-0.5-SNAPSHOT.jar -O akela-0.5-SNAPSHOT.jar`