001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.net.Authenticator.RequestorType; 009import java.net.HttpURLConnection; 010import java.net.MalformedURLException; 011import java.net.URL; 012import java.util.List; 013 014import javax.xml.parsers.ParserConfigurationException; 015 016import org.openstreetmap.josm.data.gpx.GpxData; 017import org.openstreetmap.josm.data.notes.Note; 018import org.openstreetmap.josm.data.osm.DataSet; 019import org.openstreetmap.josm.gui.progress.ProgressMonitor; 020import org.openstreetmap.josm.io.auth.CredentialsAgentException; 021import org.openstreetmap.josm.io.auth.CredentialsManager; 022import org.openstreetmap.josm.tools.HttpClient; 023import org.openstreetmap.josm.tools.Logging; 024import org.openstreetmap.josm.tools.Utils; 025import org.openstreetmap.josm.tools.XmlParsingException; 026import org.openstreetmap.josm.tools.XmlUtils; 027import org.w3c.dom.Document; 028import org.w3c.dom.Node; 029import org.xml.sax.SAXException; 030 031/** 032 * This DataReader reads directly from the REST API of the osm server. 033 * 034 * It supports plain text transfer as well as gzip or deflate encoded transfers; 035 * if compressed transfers are unwanted, set property osm-server.use-compression 036 * to false. 037 * 038 * @author imi 039 */ 040public abstract class OsmServerReader extends OsmConnection { 041 private final OsmApi api = OsmApi.getOsmApi(); 042 private boolean doAuthenticate; 043 protected boolean gpxParsedProperly; 044 protected String contentType; 045 046 /** 047 * Constructs a new {@code OsmServerReader}. 048 */ 049 protected OsmServerReader() { 050 try { 051 doAuthenticate = OsmApi.isUsingOAuth() 052 && CredentialsManager.getInstance().lookupOAuthAccessToken() != null 053 && OsmApi.USE_OAUTH_FOR_ALL_REQUESTS.get(); 054 } catch (CredentialsAgentException e) { 055 Logging.warn(e); 056 } 057 } 058 059 /** 060 * Open a connection to the given url and return a reader on the input stream 061 * from that connection. In case of user cancel, return <code>null</code>. 062 * Relative URL's are directed to API base URL. 063 * @param urlStr The url to connect to. 064 * @param progressMonitor progress monitoring and abort handler 065 * @return A reader reading the input stream (servers answer) or <code>null</code>. 066 * @throws OsmTransferException if data transfer errors occur 067 */ 068 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException { 069 return getInputStream(urlStr, progressMonitor, null); 070 } 071 072 /** 073 * Open a connection to the given url and return a reader on the input stream 074 * from that connection. In case of user cancel, return <code>null</code>. 075 * Relative URL's are directed to API base URL. 076 * @param urlStr The url to connect to. 077 * @param progressMonitor progress monitoring and abort handler 078 * @param reason The reason to show on console. Can be {@code null} if no reason is given 079 * @return A reader reading the input stream (servers answer) or <code>null</code>. 080 * @throws OsmTransferException if data transfer errors occur 081 */ 082 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException { 083 try { 084 api.initialize(progressMonitor); 085 String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr); 086 return getInputStreamRaw(url, progressMonitor, reason); 087 } finally { 088 progressMonitor.invalidate(); 089 } 090 } 091 092 /** 093 * Return the base URL for relative URL requests 094 * @return base url of API 095 */ 096 protected String getBaseUrl() { 097 return api.getBaseUrl(); 098 } 099 100 /** 101 * Open a connection to the given url and return a reader on the input stream 102 * from that connection. In case of user cancel, return <code>null</code>. 103 * @param urlStr The exact url to connect to. 104 * @param progressMonitor progress monitoring and abort handler 105 * @return An reader reading the input stream (servers answer) or <code>null</code>. 106 * @throws OsmTransferException if data transfer errors occur 107 */ 108 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException { 109 return getInputStreamRaw(urlStr, progressMonitor, null); 110 } 111 112 /** 113 * Open a connection to the given url and return a reader on the input stream 114 * from that connection. In case of user cancel, return <code>null</code>. 115 * @param urlStr The exact url to connect to. 116 * @param progressMonitor progress monitoring and abort handler 117 * @param reason The reason to show on console. Can be {@code null} if no reason is given 118 * @return An reader reading the input stream (servers answer) or <code>null</code>. 119 * @throws OsmTransferException if data transfer errors occur 120 */ 121 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException { 122 return getInputStreamRaw(urlStr, progressMonitor, reason, false); 123 } 124 125 /** 126 * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream 127 * from that connection. In case of user cancel, return <code>null</code>. 128 * @param urlStr The exact url to connect to. 129 * @param progressMonitor progress monitoring and abort handler 130 * @param reason The reason to show on console. Can be {@code null} if no reason is given 131 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition} 132 * for {@code filename} and uncompress a gzip/bzip2/xz/zip stream. 133 * @return An reader reading the input stream (servers answer) or <code>null</code>. 134 * @throws OsmTransferException if data transfer errors occur 135 */ 136 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 137 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 138 return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null); 139 } 140 141 /** 142 * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream 143 * from that connection. In case of user cancel, return <code>null</code>. 144 * @param urlStr The exact url to connect to. 145 * @param progressMonitor progress monitoring and abort handler 146 * @param reason The reason to show on console. Can be {@code null} if no reason is given 147 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition} 148 * for {@code filename} and uncompress a gzip/bzip2/xz/zip stream. 149 * @param httpMethod HTTP method ("GET", "POST" or "PUT") 150 * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method. 151 * @return An reader reading the input stream (servers answer) or <code>null</code>. 152 * @throws OsmTransferException if data transfer errors occur 153 * @since 12596 154 */ 155 @SuppressWarnings("resource") 156 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 157 boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException { 158 try { 159 if (NetworkManager.isOffline(urlStr)) { 160 throw new OsmApiException(OfflineAccessException.forResource(urlStr)); 161 } 162 163 URL url = null; 164 try { 165 url = new URL(urlStr.replace(" ", "%20")); 166 } catch (MalformedURLException e) { 167 throw new OsmTransferException(e); 168 } 169 170 String protocol = url.getProtocol(); 171 if ("file".equals(protocol) || "jar".equals(protocol)) { 172 try { 173 return Utils.openStream(url); 174 } catch (IOException e) { 175 throw new OsmTransferException(e); 176 } 177 } 178 179 final HttpClient client = HttpClient.create(url, httpMethod) 180 .setAccept("application/xml, */*;q=0.8") 181 .setFinishOnCloseOutput(false) 182 .setReasonForRequest(reason) 183 .setOutputMessage(tr("Downloading data...")) 184 .setRequestBody(requestBody); 185 activeConnection = client; 186 adaptRequest(client); 187 if (doAuthenticate) { 188 addAuth(client); 189 } 190 if (cancel) 191 throw new OsmTransferCanceledException("Operation canceled"); 192 193 final HttpClient.Response response; 194 try { 195 response = client.connect(progressMonitor); 196 contentType = response.getContentType(); 197 } catch (IOException e) { 198 Logging.error(e); 199 OsmTransferException ote = new OsmTransferException( 200 tr("Could not connect to the OSM server. Please check your internet connection."), e); 201 ote.setUrl(url.toString()); 202 throw ote; 203 } 204 try { 205 if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) { 206 CredentialsManager.getInstance().purgeCredentialsCache(RequestorType.SERVER); 207 throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null); 208 } 209 210 if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH) 211 throw new OsmTransferCanceledException("Proxy Authentication Required"); 212 213 if (response.getResponseCode() != HttpURLConnection.HTTP_OK) { 214 String errorHeader = response.getHeaderField("Error"); 215 String errorBody = fetchResponseText(response); 216 throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString(), null, 217 contentType); 218 } 219 220 response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition); 221 return response.getContent(); 222 } catch (OsmTransferException e) { 223 throw e; 224 } catch (IOException e) { 225 throw new OsmTransferException(e); 226 } 227 } finally { 228 progressMonitor.invalidate(); 229 } 230 } 231 232 private static String fetchResponseText(final HttpClient.Response response) { 233 try { 234 return response.fetchContent(); 235 } catch (IOException e) { 236 Logging.error(e); 237 return tr("Reading error text failed."); 238 } 239 } 240 241 /** 242 * Allows subclasses to modify the request. 243 * @param request the prepared request 244 * @since 9308 245 */ 246 protected void adaptRequest(HttpClient request) { 247 } 248 249 /** 250 * Download OSM files from somewhere 251 * @param progressMonitor The progress monitor 252 * @return The corresponding dataset 253 * @throws OsmTransferException if any error occurs 254 */ 255 public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException; 256 257 /** 258 * Download compressed OSM files from somewhere 259 * @param progressMonitor The progress monitor 260 * @param compression compression to use 261 * @return The corresponding dataset 262 * @throws OsmTransferException if any error occurs 263 * @since 13352 264 */ 265 public DataSet parseOsm(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 266 throw new UnsupportedOperationException(); 267 } 268 269 /** 270 * Download OSM Change uncompressed files from somewhere 271 * @param progressMonitor The progress monitor 272 * @return The corresponding dataset 273 * @throws OsmTransferException if any error occurs 274 */ 275 public DataSet parseOsmChange(ProgressMonitor progressMonitor) throws OsmTransferException { 276 return null; 277 } 278 279 /** 280 * Download OSM Change compressed files from somewhere 281 * @param progressMonitor The progress monitor 282 * @param compression compression to use 283 * @return The corresponding dataset 284 * @throws OsmTransferException if any error occurs 285 * @since 13352 286 */ 287 public DataSet parseOsmChange(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 288 throw new UnsupportedOperationException(); 289 } 290 291 /** 292 * Retrieve raw gps waypoints from the server API. 293 * @param progressMonitor The progress monitor 294 * @return The corresponding GPX tracks 295 * @throws OsmTransferException if any error occurs 296 */ 297 public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException { 298 return null; 299 } 300 301 /** 302 * Retrieve compressed GPX files from somewhere. 303 * @param progressMonitor The progress monitor 304 * @param compression compression to use 305 * @return The corresponding GPX tracks 306 * @throws OsmTransferException if any error occurs 307 * @since 13352 308 */ 309 public GpxData parseRawGps(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 310 throw new UnsupportedOperationException(); 311 } 312 313 /** 314 * Returns true if this reader is adding authentication credentials to the read 315 * request sent to the server. 316 * 317 * @return true if this reader is adding authentication credentials to the read 318 * request sent to the server 319 */ 320 public boolean isDoAuthenticate() { 321 return doAuthenticate; 322 } 323 324 /** 325 * Sets whether this reader adds authentication credentials to the read 326 * request sent to the server. 327 * 328 * @param doAuthenticate true if this reader adds authentication credentials to the read 329 * request sent to the server 330 */ 331 public void setDoAuthenticate(boolean doAuthenticate) { 332 this.doAuthenticate = doAuthenticate; 333 } 334 335 /** 336 * Determines if the GPX data has been parsed properly. 337 * @return true if the GPX data has been parsed properly, false otherwise 338 * @see GpxReader#parse 339 */ 340 public final boolean isGpxParsedProperly() { 341 return gpxParsedProperly; 342 } 343 344 /** 345 * Downloads notes from the API, given API limit parameters 346 * 347 * @param noteLimit How many notes to download. 348 * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes. 349 * @param progressMonitor Progress monitor for user feedback 350 * @return List of notes returned by the API 351 * @throws OsmTransferException if any errors happen 352 */ 353 public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException { 354 return null; 355 } 356 357 /** 358 * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added 359 * 360 * @param progressMonitor progress monitor 361 * @return A list of notes parsed from the URL 362 * @throws OsmTransferException if any error occurs during dialog with OSM API 363 */ 364 public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException { 365 return null; 366 } 367 368 /** 369 * Download notes from a URL that contains a compressed notes dump file 370 * @param progressMonitor progress monitor 371 * @param compression compression to use 372 * @return A list of notes parsed from the URL 373 * @throws OsmTransferException if any error occurs during dialog with OSM API 374 * @since 13352 375 */ 376 public List<Note> parseRawNotes(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 377 throw new UnsupportedOperationException(); 378 } 379 380 /** 381 * Returns an attribute from the given DOM node. 382 * @param node DOM node 383 * @param name attribute name 384 * @return attribute value for the given attribute 385 * @since 12510 386 */ 387 protected static String getAttribute(Node node, String name) { 388 return node.getAttributes().getNamedItem(name).getNodeValue(); 389 } 390 391 /** 392 * DOM document parser. 393 * @param <R> resulting type 394 * @since 12510 395 */ 396 @FunctionalInterface 397 protected interface DomParser<R> { 398 /** 399 * Parses a given DOM document. 400 * @param doc DOM document 401 * @return parsed data 402 * @throws XmlParsingException if an XML parsing error occurs 403 */ 404 R parse(Document doc) throws XmlParsingException; 405 } 406 407 /** 408 * Fetches generic data from the DOM document resulting an API call. 409 * @param api the OSM API call 410 * @param subtask the subtask translated message 411 * @param parser the parser converting the DOM document (OSM API result) 412 * @param <T> data type 413 * @param monitor The progress monitor 414 * @param reason The reason to show on console. Can be {@code null} if no reason is given 415 * @return The converted data 416 * @throws OsmTransferException if something goes wrong 417 * @since 12510 418 */ 419 public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason) 420 throws OsmTransferException { 421 try { 422 monitor.beginTask(""); 423 monitor.indeterminateSubTask(subtask); 424 try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) { 425 return parser.parse(XmlUtils.parseSafeDOM(in)); 426 } 427 } catch (OsmTransferException e) { 428 throw e; 429 } catch (IOException | ParserConfigurationException | SAXException e) { 430 throw new OsmTransferException(e); 431 } finally { 432 monitor.finishTask(); 433 } 434 } 435}