i had a program that could get almost any web address and save the file it returned, but it didnt work for php generated addresses. the reason was that my code depended on there being a contentLength attribute, and there is none generated for things like this (like a google search). so i used an if statement to test weather there was one, and if there wasnt, instead of using a buffered read, i tried to read the contents one byte at a time. i had to use a vector for this because again, the contentLength wasnt known. now im stuck with a vector and i dont know how to get it into the file.
import java.net.*; import java.io.*; import javax.swing.*; import java.awt.*; import java.util.Vector; public class SourceViewer3 { public static void main (String[] args) throws Exception { for (int i = 0; i < 1; i++)//args.length; i++) { try { URL u = new URL("http://www.google.com/#hl=en&source=hp&q=unreal&aq=f&aqi=g10&aql=&oq=&gs_rfai=&fp=84c7fb41710deb10"); HttpURLConnection uc = (HttpURLConnection)u.openConnection( ); int code = uc.getResponseCode( ); String response = uc.getResponseMessage( ); System.out.println("HTTP/1.x " + code + " " + response); for (int j = 1; ; j++) { String header = uc.getHeaderField(j); String key = uc.getHeaderFieldKey(j); if (header == null || key == null) break; System.out.println(uc.getHeaderFieldKey(j) + ": " + header); } String contentType = uc.getContentType(); int contentLength = uc.getContentLength(); System.out.println(contentType + contentLength); InputStream in = new BufferedInputStream(uc.getInputStream( )); if (contentLength == -1) { System.out.println("Alert: No contentLength, must be php or something, proceeding with slower bit by bit read\n."); Reader r = new InputStreamReader(in); int c; Vector<Object> data = new Vector<Object>(); contentLength = 0; while ((c = r.read( )) != -1) { System.out.print((Object) c); data.add(c); contentLength++; } in.close(); String dirText = "text"; String dirTarget = "target"; String filename = "000" + u.getFile().substring(u.getFile().lastIndexOf('/') + 1); String dirAndFile; String dirNoSlashes = "run_from_main"; // determine weather the file is a target or is to be parsed for more files if(contentType.startsWith("text/")) { dirAndFile = dirNoSlashes + "\\" + dirText + "\\" + filename; } else { dirAndFile = dirNoSlashes + "\\" + dirTarget + "\\" + filename; } FileOutputStream out = new FileOutputStream(dirAndFile); byte[] data1 = (byte[])data.toArray(); out.write(data1); out.flush(); out.close(); } else { byte[] data = new byte[contentLength]; int bytesRead = 0; int offset = 0; while (offset < contentLength) { bytesRead = in.read(data, offset, data.length - offset); if (bytesRead == -1) { break; } offset += bytesRead; System.out.println("bytes read ="+ bytesRead +"\n"); } in.close(); if (offset != contentLength) { throw new IOException("Only read " + offset + " bytes; Expected " + contentLength + " bytes"); } String dirText = "text"; String dirTarget = "target"; String filename = "000" + u.getFile().substring(u.getFile().lastIndexOf('/') + 1); String dirAndFile; String dirNoSlashes = "run_from_main"; // determine weather the file is a target or is to be parsed for more files if(contentType.startsWith("text/")) { dirAndFile = dirNoSlashes + "\\" + dirText + "\\" + filename; } else { dirAndFile = dirNoSlashes + "\\" + dirTarget + "\\" + filename; } FileOutputStream out = new FileOutputStream(dirAndFile); out.write(data); out.flush(); out.close(); } } catch (MalformedURLException ex) { System.err.println(args[0] + " is not a parseable URL"); } catch (IOException ex) { System.err.println(ex); } } } }