First commit of spavro

.gitignore

@@ -0,0 +1,28 @@
+# Standard Python files
+*.template.py
+*.pyc
+#*#
+._*
+*.sqlite
+*.pyc
+*.pid
+*.log
+tmp/
+src/
+build/
+logs/*
+
+# OSX file metadata
+.DS_Store
+
+environment.py
+environments/*
+
+# Unit test artifacts
+.coverage_output.xml
+.nose_output.xml
+.coverage
+
+# other stuff
+samples
+output

VERSION.txt

@@ -0,0 +1 @@
+1.9.0-SNAPSHOT

avro.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 1.0
+Name: avro
+Version: -AVRO-VERSION-
+Summary: Avro is a serialization and RPC framework.
+Home-page: http://avro.apache.org/
+Author: Apache Avro
+Author-email: [email protected]
+License: Apache License 2.0
+Description: UNKNOWN
+Keywords: avro serialization rpc
+Platform: UNKNOWN

avro.egg-info/SOURCES.txt

@@ -0,0 +1,27 @@
+setup.py
+./scripts/avro
+avro.egg-info/PKG-INFO
+avro.egg-info/SOURCES.txt
+avro.egg-info/dependency_links.txt
+avro.egg-info/requires.txt
+avro.egg-info/top_level.txt
+src/avro/LICENSE
+src/avro/NOTICE
+src/avro/__init__.py
+src/avro/datafile.py
+src/avro/io.py
+src/avro/ipc.py
+src/avro/protocol.py
+src/avro/schema.py
+src/avro/tool.py
+src/avro/txipc.py
+test/test_datafile.py
+test/test_datafile_interop.py
+test/test_io.py
+test/test_ipc.py
+test/test_protocol.py
+test/test_schema.py
+test/test_script.py
+test/test_tether_task.py
+test/test_tether_task_runner.py
+test/test_tether_word_count.py

avro.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

avro.egg-info/requires.txt

@@ -0,0 +1,3 @@
+
+[snappy]
+python-snappy

avro.egg-info/top_level.txt

@@ -0,0 +1 @@
+avro

build.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e # exit on error
+
+function usage {
+  echo "Usage: $0 {test|dist|clean}"
+  exit 1
+}
+
+if [ $# -eq 0 ]
+then
+  usage
+fi
+
+if [ -f VERSION.txt ]
+then
+  VERSION=`cat VERSION.txt`
+else
+  VERSION=`cat ../../share/VERSION.txt`
+fi
+
+for target in "$@"
+do
+
+case "$target" in
+  test)
+    ant test
+    ;;
+
+  dist)
+     ant dist
+    ;;
+
+  clean)
+    ant clean
+    rm -rf userlogs/
+    ;;
+
+  *)
+    usage
+esac
+
+done
+
+exit 0

build.xml

@@ -0,0 +1,235 @@
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="Avro" default="dist" xmlns:ivy="antlib:org.apache.ivy.ant">
+
+  <!-- Load user's default properties. -->
+  <property file="${user.home}/build.properties"/>
+
+  <!-- Shared directories -->
+  <property name="share.dir" value="${basedir}/../../share"/>
+  <property name="share.schema.dir" value="${share.dir}/schemas/"/>
+  <property name="dist.dir" value="${basedir}/../../dist/py"/>
+  <property name="top.build" value="${basedir}/../../build"/>
+  <property name="interop.data.dir" value="${top.build}/interop/data"/>
+
+  <property name="python" value="python"/>
+
+  <!-- Python implementation directories -->
+  <property name="build.dir" value="${basedir}/build"/>
+  <property name="src.dir" value="${basedir}/src"/>
+  <property name="lib.dir" value="${basedir}/lib"/>
+  <property name="test.dir" value="${basedir}/test"/>
+
+  <property name="ivy.version" value="2.2.0"/>
+  <property name="ivy.jar" value="${basedir}/lib/ivy-${ivy.version}.jar"/>
+
+  <!-- Load shared properties -->
+  <loadfile srcFile="${share.dir}/VERSION.txt" property="avro.version">
+    <filterchain>
+      <striplinebreaks/>
+    </filterchain>
+  </loadfile>
+  <loadfile srcFile="${share.schema.dir}/org/apache/avro/ipc/HandshakeRequest.avsc" property="handshake.request.json"/>
+  <loadfile srcFile="${share.schema.dir}/org/apache/avro/ipc/HandshakeResponse.avsc" property="handshake.response.json"/>
+
+  <path id="java.classpath">
+    <fileset dir="lib">
+      <include name="**/*.jar" />
+    </fileset>
+  </path>
+
+  <path id="test.path">
+    <pathelement location="${build.dir}/src"/>
+    <pathelement location="${build.dir}/test"/>
+    <pathelement location="${build.dir}/lib"/>
+  </path>
+
+  <target name="init" description="Create the build directory.">
+    <mkdir dir="${build.dir}"/>
+    <available file="${ivy.jar}" property="ivy.jar.found"/>
+    <antcall target="ivy-download"/>
+    <typedef uri="antlib:org.apache.ivy.ant">
+      <classpath>
+        <pathelement location="${ivy.jar}" />
+      </classpath>
+    </typedef>
+  </target>
+
+  <target name="ivy-download" unless="ivy.jar.found" >
+    <get src="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar" dest="${ivy.jar}" usetimestamp="true" />
+  </target>
+
+  <target name="build"
+          description="Copy project files to build/ and do string replacement."
+          depends="init">
+    <!-- Copy src/, test/, lib/ -->
+    <copy todir="${build.dir}/src">
+      <fileset dir="${src.dir}">
+        <exclude name="**/*.pyc"/>
+        <exclude name="**/*.py~"/>
+      </fileset>
+    </copy>
+    <copy todir="${build.dir}/test">
+      <fileset dir="${test.dir}">
+        <exclude name="**/*.pyc"/>
+        <exclude name="**/*.py~"/>
+      </fileset>
+    </copy>
+    <copy todir="${build.dir}/lib">
+      <fileset dir="${lib.dir}" />
+    </copy>
+
+    <!--Copy the protocols used for tethering -->
+    <copy todir="${build.dir}/src/avro/tether">
+        <fileset dir="${share.schema.dir}/org/apache/avro/mapred/tether/">
+    <include name="*.avpr"/>
+        </fileset>
+    </copy>
+    <!-- Inline the handshake schemas -->
+    <copy file="${src.dir}/avro/ipc.py"
+          toFile="${build.dir}/src/avro/ipc.py"
+          overwrite="true">
+      <filterset>
+        <filter token="HANDSHAKE_REQUEST_SCHEMA"
+          value="${handshake.request.json}"/>
+        <filter token="HANDSHAKE_RESPONSE_SCHEMA"
+          value="${handshake.response.json}"/>
+     </filterset>
+    </copy>
+
+    <!-- Inline the Avro version -->
+    <copy file="${basedir}/setup.py"
+          toFile="${build.dir}/setup.py"
+          overwrite="true">
+      <filterset>
+        <filter token="AVRO_VERSION" value="${avro.version}"/>
+      </filterset>
+    </copy>
+
+     <!-- Inline the Avro version -->
+     <copy file="${basedir}/scripts/avro"
+           toFile="${build.dir}/scripts/avro"
+           overwrite="true">
+       <filterset>
+         <filter token="AVRO_VERSION" value="${avro.version}"/>
+       </filterset>
+     </copy>
+     <!-- Make executable (Ant does not preseve executable bit) -->
+     <exec executable="chmod">
+         <arg value="a+x" />
+         <arg value="${build.dir}/scripts/avro" />
+     </exec>
+
+    <!-- Inline the interop data directory -->
+    <copy file="${test.dir}/test_datafile_interop.py"
+          toFile="${build.dir}/test/test_datafile_interop.py"
+          overwrite="true">
+      <filterset>
+        <filter token="INTEROP_DATA_DIR" value="${interop.data.dir}"/>
+      </filterset>
+    </copy>
+
+    <!-- Ensure we have a local copy of the tools jar -->
+    <ivy:retrieve
+        pattern="${basedir}/../java/tools/target/[artifact]-[revision].[ext]"/>
+
+    <!-- Inline the location of the tools jar -->
+    <copy file="${test.dir}/test_tether_word_count.py"
+          toFile="${build.dir}/test/test_tether_word_count.py"
+          overwrite="true">
+      <filterset>
+  <filter token="AVRO_VERSION" value="${avro.version}"/>
+  <filter token="TOPDIR" value="${basedir}"/>
+      </filterset>
+    </copy>
+  </target>
+
+  <target name="test"
+          description="Run python unit tests"
+          depends="build">
+    <taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
+	     classpathref="java.classpath"/>
+    <py-test python="${python}" pythonpathref="test.path" >
+      <fileset dir="${build.dir}/test">
+        <include name="test_*.py"/>
+        <exclude name="test_datafile_interop.py"/>
+      </fileset>
+    </py-test>
+  </target>
+
+    <!--Created a unittest to run just the tests for tethered jobs.
+    -->
+    <target name="test-tether"
+          description="Run unit tests for a hadoop python-tethered job."
+          depends="build">
+    <taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
+       classpathref="java.classpath"/>
+    <py-test python="${python}" pythonpathref="test.path">
+      <fileset dir="${build.dir}/test">
+        <include name="test_tether*.py"/>
+        <!--<exclude name="test_datafile_interop.py"/>-->
+      </fileset>
+    </py-test>
+  </target>
+
+
+  <target name="interop-data-test"
+          description="Run python interop data tests"
+          depends="build">
+    <taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
+	     classpathref="java.classpath"/>
+    <py-test python="${python}" pythonpathref="test.path" >
+      <fileset dir="${build.dir}/test">
+        <include name="test_datafile_interop.py"/>
+      </fileset>
+    </py-test>
+  </target>
+
+  <target name="interop-data-generate"
+          description="Generate Python interop data files."
+          depends="build">
+    <mkdir dir="${interop.data.dir}"/>
+    <exec executable="${python}">
+      <env key="PYTHONPATH" value="$PYTHONPATH:${build.dir}/src"/>
+      <arg value="${build.dir}/test/gen_interop_data.py"/>
+      <arg value="${share.dir}/test/schemas/interop.avsc"/>
+      <arg value="${interop.data.dir}/py.avro"/>
+    </exec>
+  </target>
+
+  <target name="dist"
+          description="Build source distribution"
+          depends="build">
+    <mkdir dir="${dist.dir}"/>
+    <exec executable="${python}" failonerror="true" dir="${build.dir}">
+      <arg value="${build.dir}/setup.py"/>
+      <arg value="sdist"/>
+      <arg value="--dist-dir=${dist.dir}"/>
+    </exec>
+  </target>
+
+  <target name="clean"
+          description="Delete build files and their directories">
+    <delete includeemptydirs="true" failonerror="false">
+      <fileset file="MANIFEST"/>
+      <fileset dir="${build.dir}"/>
+    </delete>
+  </target>
+
+</project>

ipc/HandshakeRequest.avsc

@@ -0,0 +1,11 @@
+{
+    "type": "record",
+    "name": "HandshakeRequest", "namespace":"org.apache.avro.ipc",
+    "fields": [
+        {"name": "clientHash",
+	 "type": {"type": "fixed", "name": "MD5", "size": 16}},
+        {"name": "clientProtocol", "type": ["null", "string"]},
+        {"name": "serverHash", "type": "MD5"},
+ 	{"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]}
+ ]
+}