highcharts 1.5 ExportController.java using servlet-api-2.5

Recently we have integrated Moxie Group’s GWT Highchart in our project and ran into the conflict with ExportController’s servlet-api-3.0 multi-part issues.

Like the old GWT does, it uses and includes servlet-api-2.5 or rather jetty 7.  Fortunately our project has already got apache-fileupload in its dependency and we could use the fileitemiterator to read the multi-part items. Changes are made in processrequest(HttpServletRequest request, HttpServletResponse response) and getParameter(HttpServletRequest request, String name) Here comes the modified code:

package com.highcharts.export.controller;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.Enumeration;

import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.batik.transcoder.TranscoderException;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.log4j.Logger;
import org.mortbay.log.Log;

import com.highcharts.export.util.MimeType;
import com.highcharts.export.util.SVGRasterizer;
import com.highcharts.export.util.SVGRasterizerException;

public class ExportController extends HttpServlet
{
private static final long serialVersionUID = 1L;

private static final String REQUEST_METHOD_POST = "POST";

private static final String CONTENT_TYPE_MULTIPART = "multipart/";

private static final String FORBIDDEN_WORD = "<!ENTITY";

protected static Logger logger = Logger.getLogger("exportservlet");

public ExportController()
{
super();
}

public void init()
{
}

protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
processrequest(request, response);
}

protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
{
processrequest(request, response);
}

public void processrequest(HttpServletRequest request, HttpServletResponse response) throws IOException,
ServletException
{
String svg = getParameter(request, "svg");
String filename = getFilename(getParameter(request, "filename"));
Float width = getWidth(getParameter(request, "width"));
MimeType mime = getMime(getParameter(request, "type"));

try
{
boolean multi = isMultipartRequest(request);

if (multi)
{
try
{
final FileItemFactory factory = new DiskFileItemFactory();
final ServletFileUpload upload = new ServletFileUpload(factory);
final FileItemIterator filesIterator = upload.getItemIterator(request);
while (filesIterator.hasNext())
{
final FileItemStream item = filesIterator.next();
final String name = item.getFieldName();
if ("svg".equals(name))
{
svg = new String(toByteArray(item), "UTF-8");
} else if ("filename".equals(name))
{
filename = getFilename(new String(toByteArray(item), "UTF-8"));
} else if ("width".equals(name))
{
width = getWidth(new String(toByteArray(item), "UTF-8"));
} else if ("type".equals(name))
{
mime = getMime(new String(toByteArray(item), "UTF-8"));
}
}
} catch (Exception e)
{
Log.warn("Failed to parse multi-part parameter: " + e);
}
} else
{

}
if (svg == null || svg.isEmpty())
{
throw new ServletException("The required - svg - post parameter is missing");
}
if (svg.indexOf(FORBIDDEN_WORD) > -1 || svg.indexOf(FORBIDDEN_WORD.toLowerCase()) > -1)
{
throw new ServletException("The - svg - post parameter could contain a malicious attack");
}

ExportController.writeFileContentToHttpResponse(svg, filename, width, mime, response);

} catch (IOException ioe)
{
logger.error("Oops something happened here redirect to error-page, " + ioe.getMessage());
sendError(request, response, ioe);
} catch (ServletException sce)
{
logger.error("Oops something happened here redirect to error-page, " + sce.getMessage());
sendError(request, response, sce);
}
}

public static byte[] toByteArray(final FileItemStream item) throws IOException
{
final InputStream in = item.openStream();
final ByteArrayOutputStream out = new ByteArrayOutputStream();
copy(in, out);
final byte[] byteArray = out.toByteArray();
in.close();
return byteArray;
}

public static void copy(final InputStream in, final OutputStream out) throws IOException
{
final byte[] buf = new byte[1024 * 8];
while (true)
{
final int length = in.read(buf);
if (length == -1)
{
break;
}
out.write(buf, 0, length);
}
out.flush();
}

/*
* Util methods
*/

public static void writeFileContentToHttpResponse(String svg, String filename, Float width, MimeType mime,
HttpServletResponse response) throws IOException, ServletException
{

ByteArrayOutputStream stream = new ByteArrayOutputStream();

if (!MimeType.SVG.equals(mime))
{
try
{
stream = SVGRasterizer.getInstance().transcode(stream, svg, mime, width);
} catch (SVGRasterizerException sre)
{
logger.error("Error while transcoding svg file to an image", sre);
stream.close();
throw new ServletException("Error while transcoding svg file to an image");
} catch (TranscoderException te)
{
logger.error("Error while transcoding svg file to an image", te);
stream.close();
throw new ServletException("Error while transcoding svg file to an image");
}
} else
{
stream.write(svg.getBytes());
}

// prepare response
response.reset();
response.setContentLength(stream.size());
response.setCharacterEncoding("utf-8");
response.setHeader("Content-disposition", "attachment; filename=" + filename + "." + mime.name().toLowerCase());
response.setHeader("Content-type", mime.getType());
// set encoding before writing to out, check this
ServletOutputStream out = response.getOutputStream();
// Send content to Browser
out.write(stream.toByteArray());
out.flush();
}

public static final boolean isMultipartRequest(HttpServletRequest request)
{
// inspired by org.apache.commons.fileupload
logger.debug("content-type " + request.getContentType());
return REQUEST_METHOD_POST.equalsIgnoreCase(request.getMethod()) && request.getContentType() != null
&& request.getContentType().toLowerCase().startsWith(CONTENT_TYPE_MULTIPART);
}

private String getParameter(HttpServletRequest request, String name) throws IOException, ServletException
{

return request.getParameter(name);

}

private String getFilename(String name)
{
return (name != null) ? name : "chart";
}

private static Float getWidth(String width)
{
if (width != null && !width.isEmpty())
{
Float parsedWidth = Float.valueOf(width);
if (parsedWidth.compareTo(0.0F) > 0)
{
return parsedWidth;
}
}
return null;
}

private static MimeType getMime(String mime)
{
MimeType type = MimeType.get(mime);
if (type != null)
{
return type;
}
return MimeType.PNG;
}

protected void sendError(HttpServletRequest request, HttpServletResponse response, Throwable ex) throws IOException,
ServletException
{
String headers = null;
String htmlHeader =
"<HTML><HEAD><TITLE>Highcharts Export error</TITLE><style type=\"text/css\">"
+ "body {font-family: \"Trebuchet MS\", Arial, Helvetica, sans-serif;} table {border-collapse: collapse;}th {background-color:green;color:white;} td, th {border: 1px solid #98BF21;} </style></HEAD><BODY>";
String htmlFooter = "</BODY></HTML>";

response.setContentType("text/html");

PrintWriter out = response.getWriter();
Enumeration<String> e = request.getHeaderNames();
String svg = this.getParameter(request, "svg");

out.println(htmlHeader);
out.println("<h3>Error while converting SVG</h3>");
out.println("<h4>Error message</h4>");
out.println("<p>" + ex.getMessage() + "</p>");
out.println("<h4>Debug steps</h4><ol>"
+ "<li>Copy the SVG:<br/><textarea cols=100 rows=5>"
+ svg
+ "</textarea></li>"
+ "<li>Go to <a href='http://validator.w3.org/#validate_by_input' target='_blank'>validator.w3.org/#validate_by_input</a></li>"
+ "<li>Paste the SVG</li>" + "<li>Click More Options and select SVG 1.1 for Use Doctype</li>"
+ "<li>Click the Check button</li></ol>");

out.println("<h4>Request Headers</h4>");
out.println("<TABLE>");
out.println("<tr><th> Header </th><th> Value </th>");

while (e.hasMoreElements())
{
headers = (String) e.nextElement();
if (headers != null)
{
out.println("<tr><td align=center><b>" + headers + "</td>");
out.println("<td align=center>" + request.getHeader(headers) + "</td></tr>");
}
}
out.println("</TABLE><BR>");
out.println(htmlFooter);

}
}
<pre>
Advertisements

public online sources of financial and economic data

World

  • OECD.StatExtracts includes data and metadata for OECD countries and selected non-member economies.
  • EuroNext. Bonds and Equities are available. “Search by Criteria” -> select instrument -> “Data downloads”.

United Kingdom

United States


Foreign Exchange


Equity and Equity Indices


Fixed Income


Options and Implied Volatility


Futures


Commodities


Multiple Asset Classes and Miscellaneous

Example JSON request: The link below asks CME server to return back options data for given strikes:

http://www.cmegroup.com/CmeWS/md/MDServer/V1/Venue/G/Exchange/XCME/FOI/OPT/Product/ES?currentTime=1311084303814&contractCDs=,ESU1%20C1315,ESU1%20P1315,ESU1%20C1320,ESU1%20P1320,ESU1%20C1325,ESU1%20P1325,ESU1%20C1330,ESU1%20P1330,ESU1%20C1335,ESU1%20P1335

Example Yahoo CSV:

http://ichart.finance.yahoo.com/table.csv?s=INTC&a=06&b=9&c=1986&d=2&e=5&f=2008&g=d&ignore=.csv
s: This is where you can specify your stock quote, if you want to download stock quote for Microsoft, just enter it as ‘s=MSFT’
a: start month (’00’ is for January)
b: start day
c: start year
d: end month
e: end day
f: end year
g: interval, ‘d’ for daily, ‘w’ for weekly and ‘m’ is for monthly prices. The default is ‘daily’ if you ignore this parameter.

 

Intraday and Tick data

Google Finance

In Google Finance, intra-day data is available free for several stock markets. The complete list can be found here.

Data is available in several frequencies with the lowest one being one-minute time frame.

The URL format is: http://www.google.com/finance/getprices?i=%5BPERIOD%5D&p=%5BDAYS%5Dd&f=d,o,h,l,c,v&df=cpct&q=%5BTICKER%5D

Example: http://www.google.com/finance/getprices?i=60&p=10d&f=d,o,h,l,c,v&df=cpct&q=IBM

[PERIOD]: Interval or frequency in seconds
[DAYS]: The historical data period, where “10d” means that we need historical stock prices data for the past 10 days.
[TICKER]: This is the ticker symbol of the stock

Symbol Lookup: http://www.google.com/finance

More information: Intraday Data for US Stocks

  • The base url is http://www.google.com/finance/getprices
  • q is the symbol (AAPL)
  • x is the exchange (NASD)
  • i is the interval in seconds (120 = seconds = 2 minutes)
  • sessions is the session requested (ext_hours)
  • p is the time period (5d = 5 days)
  • f is the requested fields (d,c,v,o,h,l)
  • df ?? (cpct)
  • auto ?? (1)
  • ts is potentially a time stamp (1324323553 905)

Yahoo Finance

As with Google Finance, Yahoo allows you to download intraday data for several stock markets. You can get data for 62 exchanges.

Format : http://chartapi.finance.yahoo.com/instrument/1.0/%5BTICKER%5D/chartdata;type=quote;range=1d/csv

Example: http://chartapi.finance.yahoo.com/instrument/1.0/GOOG/chartdata;type=quote;range=1d/csv

[TICKER]: This is the ticker symbol of the security

Symbol Lookup: http://finance.yahoo.com/q?s=&ql=1

More information: More information: Intraday Quotes for Major Stock Exchanges

NetFonds

NetFonds is a Norwegian website. It is THE website you should use if you need tick and bid/ask history data for stocks listed on U.S. stock exchanges.

To get tick data:

http://hopey.netfonds.no/tradedump.php?date=20120423&paper=AAPL.O&csv_format=txt

To get bid/ask price and volume data:

http://hopey.netfonds.no/posdump.php?date=20120423&paper=AAPL.O&csv_format=txt

Type the ticker symbol next to “paper=” parameter followed by the exchange code:
NASDAQ: O
NYSE: N
AMEX: A

The data format (“date” parameter) is YYYYMMDD

The good news is that you can get historical tick data for more than 20 days.
The bad news is that … sorry there is no bad news; there is however another great news: The above link allows you to retrieve real-time tick data too (From BATS exchange).

Stooq

Two periods are proposed here: 5-Minutes and Hourly. The big advantage however is that all the data is available in one compressed file.

http://stooq.com/db/h/

You can also choose to download intra-day data for some specific dates: http://stooq.com/db/

If you do not need low period data then this website will probably become your best source of intra-day data.

Dukascopy

Dukascopy, the Swiss Forex Bank has a nice CSV DATA Export tool. You will not get data for the entire U.S. stock market but you will be able to export CSV data of several ones and for different periods (1 minute, 10 minutes, and 1 hour).

Dukascopy has also intra-day data for several currency pairs and indices (Japan Topix index, Canadian TSX Indiex, VIX, Russell 2000, Russian RTS Index, CAC 40, Futsee 100…)

http://www.dukascopy.com/swiss/english/data_feed/csv_data_export/

 

Historical stock quotes data

FinancialContent
Several websites use historical data provided by financial content.
chron.com is one of them and here is the URL to get the eod data.
http://markets.chron.com/chron/?Method=gethistoricalcsv&Month=7&Page=HISTORICAL&Year=2009
&Range=3&Ticker=[Symbol name]
Just change ‘Symbol name’ with the ticker symbol of the company that interests you.

Yahoo
Yahoo has historical quotes and dividend data for US and many international stock markets.
This is the URL that let you download these quotes.
http://ichart.finance.yahoo.com/table.csv?d=6&e=1&f=2009&g=d&a=7&b=19&c=2004
&ignore=.csv&s=[Symbol name]

You can automatically download quotes from yahoo using the following downloader Yahoo EOD historical quotes .

Google
Google let you download daily and weekly eod quotes for US, Canada, UK, China and Hong Kong stock markets.
Here is the URL to grab this data:
http://www.google.com/finance/historical?output=csv&q=%5BSymbol name]

Investopedia
Investopedia provides end of day quotes, dividends and splits data for the US market.
http://simulator.investopedia.com/stocks/historicaldata.aspx?Download=1&s=%5BSymbol name]

Quotemedia
Quotemedia has US and Canadian stocks historical data.
http://app.quotemedia.com/quotetools/getHistoryDownload.csv?&webmasterId=501&startDay=02&startMonth=02
&startYear=2002&endDay=02&endMonth=07&endYear=2009&isRanged=false&symbol=[Symbol name]

You can automatically download quotes from quotemedia using the following downloader: Quotemedia Historical Quotes.

Dukascopy
Download historical and intraday quotes data.
Click on the dukascopy link, select settings, and then click on ‘Get Data’.

Kumo swcp
Get historical data for the S&P500 stocks in one file.

AOL
Daily, weekly and monthly historical quotes that can be exported in multiple formats.
http://finance.aol.com/.module/download/pfweb/historical/%5BSymbol name]?type=2&symbol=[Symbol name]
&tf=m&gran=d&fromdate=%24%7Bhist.fromdata%7D

MSN
MSN provides eod data for the US stock exchanges.
http://moneycentral.msn.com/investor/charts/chartdl.aspx?D4=1&DD=1&MA0=0&MA1=0&CF=0&PT=7&SZ=0&D5=0&#038;
DCS=2&C1=0&C2=1&width=612&height=258&D2=0&CE=0&filedownloadbt.x=59&filedownloadbt.y=12&symbol=[Symbol name]

 

Historical intraday forex quotes data

Finam

Finam is a Russian website that allows you to get at least two months worth of one-minute Forex data.
You can use Finam to export data for 12 currency pairs, including EURUSD, EURCHF, EURJPY, EURRUB…

http://export.finam.ru/file.txt?d=d&m=5&p=2&df=1&mf=7&yf=2010&dt=22&mt=9&yt=2010&f=file&e=.txt&dtf=4&#8230;

Here is how to get Finam data automatically in QuantShare:
One-Minute Intraday Data for Currency Pairs

Forexite

If you are looking for free intraday quotes that goes back several years ago then you should use Forexite. This website provides data for 16 currency pairs and the data goes back to 2001.

Periodicity is one-minute.

URL Example: http://www.forexite.com/free_forex_quotes/2011/11/011111.zip

You can get this data directly into QuantShare by using the following item:
Forex Intraday Data

Stooq

Stooq allows you to download intraday quotes for Forex and commodities very quickly by collecting all data in a compressed file.

– Go to http://stooq.com/db/h/
– Click on the link under “5Minutes -> ASCII” to download Forex, commodities and indices quotes in a zip file

Example:
http://s.stooq.com/db/h/5_world_txt.zip

And here is the corresponding QuantShare downloader:
5-Minute Historical Intraday Data for Forex, Indices and Futures/Commodities

GAIN Capital

GAIN Capital is a leading provider of online foreign exchange trading, asset management, and B2B Forex services.

Gain Capital archive contains historic rate tick data for several currencies. The data contains the following fields: Currency Pair, Date, Bid Price and Ask Price.

Example: http://ratedata.gaincapital.com/2012/04%20April/AUD_CAD_Week1.zip

Dukascopy

Dukascopy is a Swiss Forex bank. It provides free tick data for several FX majors and crosses (Euro vs. Pound Sterling, Canadian Dollar vs. Swiss Franc, Australian Dollar vs. Yen, US Dollar vs. Singapore Dollar…)

You can download tick data here:
http://www.dukascopy.com/swiss/english/data_feed/historical/

You can also download Forex data compiled in other periods (1 minute, 10 minutes, 1 hour, 1 day, 1 week and 1 month)
http://www.dukascopy.com/swiss/english/data_feed/csv_data_export/

The above URL lets you also download quotes for US Stocks and indices.