Datatable addition

src/Quandl.jl

@@ -8,17 +8,32 @@ using Base.Dates, TimeSeries, DataFrames, Requests, CSV
 
 export quandlget,
        quandl,
+       datatable,
        quandlsearch,
        set_auth_token,
+       quandl_api_key,
        interactivequandl
 
+include("config.jl")
+include("error.jl")
 include("api.jl")
 include("timearray.jl")
 include("dataframe.jl")
+include("dataset.jl")
+include("datatable.jl")
+include("search.jl")
+
 
 # Create empty auth token if none exists
 if !isfile(joinpath(dirname(@__FILE__),"../token/auth_token"))
-    set_auth_token("")
+    # Create the token directory if needed
+    if !ispath(joinpath(dirname(@__FILE__),"../token/"))
+        mkdir(joinpath(dirname(@__FILE__),"../token/"))
+    end
+    # Write to the file
+    open(joinpath(dirname(@__FILE__),"../token/auth_token"), "w") do token_file
+        write(token_file, "")
+    end
 end
 
 end

src/api.jl

@@ -1,156 +1,53 @@
-function quandlget(id::AbstractString; order="des", rows=100, frequency="daily", transformation="none",
-                   from="", to="", format="TimeArray", api_key="", silent=false)
 
-    # verify and use API key
-    if api_key==""
-        if !ispath(joinpath(dirname(@__FILE__),"../token/"))
-            println("Note: for unlimited access, you may need to get an API key at quandl.com")
-        else
-            api_key=readstring(joinpath(dirname(@__FILE__),"../token/auth_token"))
-            if !silent
-                if api_key==""
-                    println("Note: for unlimited access, you may need to get an API key at quandl.com")
-                else
-                    println("Using API key: ", api_key)
-                end
-            end
-        end
-    end
+quandlbaseurl() = "https://www.quandl.com/api/v3"
 
-    # Create a dictionary with the Query arguments that we pass to get() function
-    query_args = Dict{Any,Any}("order" => order, "rows" => rows, "collapse" => frequency, "transform" => transformation, "api_key" => api_key)
-
-    # Ignore rows argument if start or end date range specified
-    if from != ""
-        delete!(query_args, "rows")
-        query_args["start_date"] = from
+function quandlapi(path, params::Dict)
+
+    package_version = string(Pkg.installed("Quandl"))
+    headers = Dict("Request-Source" => "julia", "Request-Source-Version" => package_version)
+    if length(Quandl.api_key()) > 0
+        headers["X-Api-Token"] = Quandl.api_key()
     end
-
-    if to != ""
-        delete!(query_args, "rows")
-        query_args["end_date"] = to
+    # query param api_key takes precedence
+    if haskey(params, "api_key")
+      headers["X-Api-Token"] = params["api_key"]
+      delete!(params, "api_key")
     end
+
+
+    # Parameters require extra formatting for datatable api
+    formatted_params = formatrequestparams(params)
+    url = "$(quandlbaseurl())/$(path)"
 
     # Get the response from Quandl's API, using Query arguments (see Response.jl README)
-    resp = get("https://www.quandl.com/api/v3/datasets/$id.csv", query = query_args)
+    resp = get(url, query = formatted_params, headers = headers)
 
-    # return Union{} in case of fetch error
     if resp.status != 200
-        println(" $(resp.status): Error executing the request.")
-        Union{}
-    else
-        # Convert the response to the right DataType
-        if format == "TimeArray"
-            timearray(resp)
-        elseif format == "DataFrame"
-            dataframe(resp)
-        else
-            # return the raw fetch if requested format is not supported
-            println("Currently only TimeArray and DataFrame formats are supported")
-            resp
-        end
+        throw(QuandlApiError(Requests.text(resp), resp.status))
     end
+    resp
 end
 
-# alias quandl/quandlget
-quandl = quandlget
-
-function quandlsearch(query::AbstractString; page=1, results=20, format="DataFrame", quiet=false)
-
-    query = replace(query, " ", "+")
-
-    # Create a dictionary with the Query arguments that we pass to get() function
-    query_args = Dict{Any,Any}("query" => query, "page" => page, "per_page" => results)
-
-    # Getting response from Quandl and parsing it
-    resp       = get("https://www.quandl.com/api/v1/datasets.json", query = query_args)
-    jsondict   = Requests.json(resp)
-    data       = jsondict["docs"]
-    totalcount = jsondict["total_count"]
-
-    # Printing summary
-    if !quiet
-        println("Returning $results results of $totalcount from page $page")
-    end
-
-    # Convert the response to the right DataType
-    if format == "DataFrame"
-        # Create an NA 1x5 DataFrame
-        df = convert(DataFrame, ["" "" "" "" ""])
-
-        # Constructiong the DataFrame
-        for elem in data
-            newline = convert(DataFrame, ["$(elem["source_code"])/$(elem["code"])" "$(elem["name"])" "$(elem["frequency"])" "$(elem["from_date"])" "$(elem["to_date"])"])
-            df = vcat(df,newline)
+# This formats Dict("qopts" => Dict("columns" => ["ticker", "date", "close"]))
+# as Dict("qopts.columns" => "ticker,date,close")
+# in order to be processed correctly by the datatable api
+function formatrequestparams(params::Dict)
+   outparams = Dict() 
+   for (k, v) in params
+        if isa(v, Dict)
+            for(k1, v1) in v
+                if isa(v1, Array)
+                    outparams["$k.$k1"] = join(v1, ",")
+                else
+                    outparams["$k.$k1"] = v1
+                end
+            end
+        elseif isa(v, Array)
+            outparams[k] = join(v, ",")
+        else
+            outparams[k] = v
         end
-
-        # Naming the columns
-        names!(df, [:Code, :Name, :Frequency, :From, :To])
-
-        # Not returning the NA line
-        return df[2:end,:]
-    elseif format == "Dict"
-        return data
-    else
-        error("Invalid $format format. Currently only DataFrame and Dict are supported. If you want this format implemented, please file an issue or submit a pull request.")
-    end
-end
-
-function interactivequandl(query::AbstractString; page="1", results=20, order="des",
-                           rows=100, frequency="daily", transformation="none", format="TimeArray",
-                           auth_token="")
-
-	# Get search results
-    searches = quandlsearch(query, page = page, results = results, format="DataFrame", quiet=true)
-
-    # Print results
-    for i in 1:size(searches, 1)
-       	print_with_color(:yellow, string(i) * " ")
-       	print_with_color(:blue, string(searches[i, :Code]))
-       	print_with_color(:black, "   From " * string(searches[i, :From]) * " to " * string(searches[i, :To]) * "\n")
-       	print_with_color(:white, "    " * string(searches[i, :Name]) * "\n")
-    end
-
-    # Print prompt
-    print_with_color(:yellow, "\n==> ")
-    print_with_color(:white, "Enter n° of the dataset to be downloaded | (N)ext page | (Q)uit\n")
-    println("-------------------------------------------------------------------")
-    print_with_color(:yellow, "==> ")
-
-    # Receive input from user
-    input = readline()[1:end-1] # Remove \n from end
-
-    # Return desired output
-    if input == "q" || input == "Q" || input == "" # Quit
-       	return nothing
-    elseif input == "n" || input == "N" # Next page
-    	  return interactivequandl(query, page = string(parse(Int, page) + 1), results = results,
-                                 format = format, auth_token = auth_token)
-    else  # Get and return result
-	      #return quandl(searches[int(input), :Code],
-        return quandl(searches[parse(Int, input), :Code],
-	                     order = order, rows = rows, frequency = frequency,
-                       transformation = transformation, format = format,
-                       api_key = auth_token)
     end
+    return outparams
 end
 
-function set_auth_token(token::AbstractString)
-
-    # Check the validity of the token
-    if length(token) != 20 && length(token) != 0
-        error("Invalid Token: must be 20 characters long or be an empty string")
-    end
-
-    # Create the token directory if needed
-    if !ispath(joinpath(dirname(@__FILE__),"../token/"))
-        mkdir(joinpath(dirname(@__FILE__),"../token/"))
-    end
-
-    # Write to the file
-    open(joinpath(dirname(@__FILE__),"../token/auth_token"), "w") do token_file
-        write(token_file, token)
-    end
-
-	  return nothing
-end

src/config.jl

@@ -0,0 +1,31 @@
+# Backwards compatible
+function set_auth_token(token::AbstractString)
+
+    # Check the validity of the token
+    if length(token) != 20 && length(token) != 0
+        error("Invalid Token: must be 20 characters long or be an empty string")
+    end
+
+    Quandl.api_key(token)
+
+    return nothing
+end
+
+global qapi_key = ""
+function api_key(api_key::AbstractString="")
+    global qapi_key
+    # New key to assign
+    if length(api_key) > 0
+        qapi_key = api_key
+
+        # Write to the file
+        open(joinpath(dirname(@__FILE__),"../token/auth_token"), "w") do token_file
+            write(token_file, api_key)
+        end
+    end
+    # if nothing assigned check for old key
+    if length(qapi_key) == 0
+        qapi_key=readstring(joinpath(dirname(@__FILE__),"../token/auth_token"))
+    end
+    return qapi_key
+end

src/dataset.jl

@@ -0,0 +1,47 @@
+function dataset(id::AbstractString; order="des", rows=100, frequency="daily", transformation="none",
+                   from="", to="", format="TimeArray", api_key="", silent=false)
+
+    # verify and use API key
+
+    # Create a dictionary with the Query arguments that we pass to get() function
+    query_args = Dict{Any,Any}("order" => order, "rows" => rows, "collapse" => frequency, "transform" => transformation)
+    if api_key != ""
+        query_args["api_key"] = api_key
+    end
+    # Ignore rows argument if start or end date range specified
+    if from != ""
+        delete!(query_args, "rows")
+        query_args["start_date"] = from
+    end
+    if to != ""
+        delete!(query_args, "rows")
+        query_args["end_date"] = to
+    end
+
+
+    # Backward compatible
+    resp = Void
+    try
+        resp = quandlapi("datasets/$id.csv", query_args)
+    catch err
+        isa(err, QuandlApiError) || rethrow(err)
+        println(" $(err.status): Error executing the request.")
+        return Union{}
+    end
+
+    # Convert the response to the right DataType
+    if format == "TimeArray"
+        timearray(resp)
+    elseif format == "DataFrame"
+        dataframe(resp)
+    else
+        # return the raw fetch if requested format is not supported
+        println("Currently only TimeArray and DataFrame formats are supported")
+        resp
+    end
+end
+
+# alias quandl/quandlget
+quandlget = dataset
+quandl = quandlget
+

src/datatable.jl

@@ -0,0 +1,15 @@
+
+function datatable(code::AbstractString; kwargs...)
+    kwargs = Dict{Any, Any}(kwargs)
+    resp = quandlapi("datatables/$code.csv", kwargs)
+    df = CSV.read(IOBuffer(Requests.text(resp)))
+
+    while haskey(resp.headers, "Cursor_ID")
+        kwargs["qopts.cursor_id"] = resp.headers["Cursor_ID"]
+        println(kwargs)
+        resp = quandlapi("datatables/$code.csv", kwargs)
+        df = [df; CSV.read(IOBuffer(Requests.text(resp)))]
+    end
+
+    return df
+end

src/error.jl

@@ -0,0 +1,5 @@
+type QuandlApiError <: Exception
+    var::String
+    status::Int64
+end
+Base.showerror(io::IO, e::QuandlApiError) = print(io, e.var)