001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.io.InputStream;
008import java.net.Authenticator.RequestorType;
009import java.net.HttpURLConnection;
010import java.net.MalformedURLException;
011import java.net.URL;
012import java.util.List;
013
014import javax.xml.parsers.ParserConfigurationException;
015
016import org.openstreetmap.josm.data.gpx.GpxData;
017import org.openstreetmap.josm.data.notes.Note;
018import org.openstreetmap.josm.data.osm.DataSet;
019import org.openstreetmap.josm.gui.progress.ProgressMonitor;
020import org.openstreetmap.josm.io.auth.CredentialsAgentException;
021import org.openstreetmap.josm.io.auth.CredentialsManager;
022import org.openstreetmap.josm.tools.HttpClient;
023import org.openstreetmap.josm.tools.Logging;
024import org.openstreetmap.josm.tools.Utils;
025import org.openstreetmap.josm.tools.XmlParsingException;
026import org.openstreetmap.josm.tools.XmlUtils;
027import org.w3c.dom.Document;
028import org.w3c.dom.Node;
029import org.xml.sax.SAXException;
030
031/**
032 * This DataReader reads directly from the REST API of the osm server.
033 *
034 * It supports plain text transfer as well as gzip or deflate encoded transfers;
035 * if compressed transfers are unwanted, set property osm-server.use-compression
036 * to false.
037 *
038 * @author imi
039 */
040public abstract class OsmServerReader extends OsmConnection {
041    private final OsmApi api = OsmApi.getOsmApi();
042    private boolean doAuthenticate;
043    protected boolean gpxParsedProperly;
044    protected String contentType;
045
046    /**
047     * Constructs a new {@code OsmServerReader}.
048     */
049    protected OsmServerReader() {
050        try {
051            doAuthenticate = OsmApi.isUsingOAuth()
052                    && CredentialsManager.getInstance().lookupOAuthAccessToken() != null
053                    && OsmApi.USE_OAUTH_FOR_ALL_REQUESTS.get();
054        } catch (CredentialsAgentException e) {
055            Logging.warn(e);
056        }
057    }
058
059    /**
060     * Open a connection to the given url and return a reader on the input stream
061     * from that connection. In case of user cancel, return <code>null</code>.
062     * Relative URL's are directed to API base URL.
063     * @param urlStr The url to connect to.
064     * @param progressMonitor progress monitoring and abort handler
065     * @return A reader reading the input stream (servers answer) or <code>null</code>.
066     * @throws OsmTransferException if data transfer errors occur
067     */
068    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
069        return getInputStream(urlStr, progressMonitor, null);
070    }
071
072    /**
073     * Open a connection to the given url and return a reader on the input stream
074     * from that connection. In case of user cancel, return <code>null</code>.
075     * Relative URL's are directed to API base URL.
076     * @param urlStr The url to connect to.
077     * @param progressMonitor progress monitoring and abort handler
078     * @param reason The reason to show on console. Can be {@code null} if no reason is given
079     * @return A reader reading the input stream (servers answer) or <code>null</code>.
080     * @throws OsmTransferException if data transfer errors occur
081     */
082    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
083        try {
084            api.initialize(progressMonitor);
085            String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr);
086            return getInputStreamRaw(url, progressMonitor, reason);
087        } finally {
088            progressMonitor.invalidate();
089        }
090    }
091
092    /**
093     * Return the base URL for relative URL requests
094     * @return base url of API
095     */
096    protected String getBaseUrl() {
097        return api.getBaseUrl();
098    }
099
100    /**
101     * Open a connection to the given url and return a reader on the input stream
102     * from that connection. In case of user cancel, return <code>null</code>.
103     * @param urlStr The exact url to connect to.
104     * @param progressMonitor progress monitoring and abort handler
105     * @return An reader reading the input stream (servers answer) or <code>null</code>.
106     * @throws OsmTransferException if data transfer errors occur
107     */
108    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
109        return getInputStreamRaw(urlStr, progressMonitor, null);
110    }
111
112    /**
113     * Open a connection to the given url and return a reader on the input stream
114     * from that connection. In case of user cancel, return <code>null</code>.
115     * @param urlStr The exact url to connect to.
116     * @param progressMonitor progress monitoring and abort handler
117     * @param reason The reason to show on console. Can be {@code null} if no reason is given
118     * @return An reader reading the input stream (servers answer) or <code>null</code>.
119     * @throws OsmTransferException if data transfer errors occur
120     */
121    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
122        return getInputStreamRaw(urlStr, progressMonitor, reason, false);
123    }
124
125    /**
126     * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream
127     * from that connection. In case of user cancel, return <code>null</code>.
128     * @param urlStr The exact url to connect to.
129     * @param progressMonitor progress monitoring and abort handler
130     * @param reason The reason to show on console. Can be {@code null} if no reason is given
131     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
132     *                                                for {@code filename} and uncompress a gzip/bzip2/xz/zip stream.
133     * @return An reader reading the input stream (servers answer) or <code>null</code>.
134     * @throws OsmTransferException if data transfer errors occur
135     */
136    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
137            boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
138        return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null);
139    }
140
141    /**
142     * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream
143     * from that connection. In case of user cancel, return <code>null</code>.
144     * @param urlStr The exact url to connect to.
145     * @param progressMonitor progress monitoring and abort handler
146     * @param reason The reason to show on console. Can be {@code null} if no reason is given
147     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
148     *                                                for {@code filename} and uncompress a gzip/bzip2/xz/zip stream.
149     * @param httpMethod HTTP method ("GET", "POST" or "PUT")
150     * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method.
151     * @return An reader reading the input stream (servers answer) or <code>null</code>.
152     * @throws OsmTransferException if data transfer errors occur
153     * @since 12596
154     */
155    @SuppressWarnings("resource")
156    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
157            boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException {
158        try {
159            if (NetworkManager.isOffline(urlStr)) {
160                throw new OsmApiException(OfflineAccessException.forResource(urlStr));
161            }
162
163            URL url = null;
164            try {
165                url = new URL(urlStr.replace(" ", "%20"));
166            } catch (MalformedURLException e) {
167                throw new OsmTransferException(e);
168            }
169
170            String protocol = url.getProtocol();
171            if ("file".equals(protocol) || "jar".equals(protocol)) {
172                try {
173                    return Utils.openStream(url);
174                } catch (IOException e) {
175                    throw new OsmTransferException(e);
176                }
177            }
178
179            final HttpClient client = HttpClient.create(url, httpMethod)
180                    .setAccept("application/xml, */*;q=0.8")
181                    .setFinishOnCloseOutput(false)
182                    .setReasonForRequest(reason)
183                    .setOutputMessage(tr("Downloading data..."))
184                    .setRequestBody(requestBody);
185            activeConnection = client;
186            adaptRequest(client);
187            if (doAuthenticate) {
188                addAuth(client);
189            }
190            if (cancel)
191                throw new OsmTransferCanceledException("Operation canceled");
192
193            final HttpClient.Response response;
194            try {
195                response = client.connect(progressMonitor);
196                contentType = response.getContentType();
197            } catch (IOException e) {
198                Logging.error(e);
199                OsmTransferException ote = new OsmTransferException(
200                        tr("Could not connect to the OSM server. Please check your internet connection."), e);
201                ote.setUrl(url.toString());
202                throw ote;
203            }
204            try {
205                if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
206                    CredentialsManager.getInstance().purgeCredentialsCache(RequestorType.SERVER);
207                    throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null);
208                }
209
210                if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH)
211                    throw new OsmTransferCanceledException("Proxy Authentication Required");
212
213                if (response.getResponseCode() != HttpURLConnection.HTTP_OK) {
214                    String errorHeader = response.getHeaderField("Error");
215                    String errorBody = fetchResponseText(response);
216                    throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString(), null,
217                            contentType);
218                }
219
220                response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition);
221                return response.getContent();
222            } catch (OsmTransferException e) {
223                throw e;
224            } catch (IOException e) {
225                throw new OsmTransferException(e);
226            }
227        } finally {
228            progressMonitor.invalidate();
229        }
230    }
231
232    private static String fetchResponseText(final HttpClient.Response response) {
233        try {
234            return response.fetchContent();
235        } catch (IOException e) {
236            Logging.error(e);
237            return tr("Reading error text failed.");
238        }
239    }
240
241    /**
242     * Allows subclasses to modify the request.
243     * @param request the prepared request
244     * @since 9308
245     */
246    protected void adaptRequest(HttpClient request) {
247    }
248
249    /**
250     * Download OSM files from somewhere
251     * @param progressMonitor The progress monitor
252     * @return The corresponding dataset
253     * @throws OsmTransferException if any error occurs
254     */
255    public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException;
256
257    /**
258     * Download compressed OSM files from somewhere
259     * @param progressMonitor The progress monitor
260     * @param compression compression to use
261     * @return The corresponding dataset
262     * @throws OsmTransferException if any error occurs
263     * @since 13352
264     */
265    public DataSet parseOsm(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
266        throw new UnsupportedOperationException();
267    }
268
269    /**
270     * Download OSM Change uncompressed files from somewhere
271     * @param progressMonitor The progress monitor
272     * @return The corresponding dataset
273     * @throws OsmTransferException if any error occurs
274     */
275    public DataSet parseOsmChange(ProgressMonitor progressMonitor) throws OsmTransferException {
276        return null;
277    }
278
279    /**
280     * Download OSM Change compressed files from somewhere
281     * @param progressMonitor The progress monitor
282     * @param compression compression to use
283     * @return The corresponding dataset
284     * @throws OsmTransferException if any error occurs
285     * @since 13352
286     */
287    public DataSet parseOsmChange(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
288        throw new UnsupportedOperationException();
289    }
290
291    /**
292     * Retrieve raw gps waypoints from the server API.
293     * @param progressMonitor The progress monitor
294     * @return The corresponding GPX tracks
295     * @throws OsmTransferException if any error occurs
296     */
297    public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException {
298        return null;
299    }
300
301    /**
302     * Retrieve compressed GPX files from somewhere.
303     * @param progressMonitor The progress monitor
304     * @param compression compression to use
305     * @return The corresponding GPX tracks
306     * @throws OsmTransferException if any error occurs
307     * @since 13352
308     */
309    public GpxData parseRawGps(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
310        throw new UnsupportedOperationException();
311    }
312
313    /**
314     * Returns true if this reader is adding authentication credentials to the read
315     * request sent to the server.
316     *
317     * @return true if this reader is adding authentication credentials to the read
318     * request sent to the server
319     */
320    public boolean isDoAuthenticate() {
321        return doAuthenticate;
322    }
323
324    /**
325     * Sets whether this reader adds authentication credentials to the read
326     * request sent to the server.
327     *
328     * @param doAuthenticate  true if  this reader adds authentication credentials to the read
329     * request sent to the server
330     */
331    public void setDoAuthenticate(boolean doAuthenticate) {
332        this.doAuthenticate = doAuthenticate;
333    }
334
335    /**
336     * Determines if the GPX data has been parsed properly.
337     * @return true if the GPX data has been parsed properly, false otherwise
338     * @see GpxReader#parse
339     */
340    public final boolean isGpxParsedProperly() {
341        return gpxParsedProperly;
342    }
343
344    /**
345     * Downloads notes from the API, given API limit parameters
346     *
347     * @param noteLimit How many notes to download.
348     * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes.
349     * @param progressMonitor Progress monitor for user feedback
350     * @return List of notes returned by the API
351     * @throws OsmTransferException if any errors happen
352     */
353    public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException {
354        return null;
355    }
356
357    /**
358     * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added
359     *
360     * @param progressMonitor progress monitor
361     * @return A list of notes parsed from the URL
362     * @throws OsmTransferException if any error occurs during dialog with OSM API
363     */
364    public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException {
365        return null;
366    }
367
368    /**
369     * Download notes from a URL that contains a compressed notes dump file
370     * @param progressMonitor progress monitor
371     * @param compression compression to use
372     * @return A list of notes parsed from the URL
373     * @throws OsmTransferException if any error occurs during dialog with OSM API
374     * @since 13352
375     */
376    public List<Note> parseRawNotes(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
377        throw new UnsupportedOperationException();
378    }
379
380    /**
381     * Returns an attribute from the given DOM node.
382     * @param node DOM node
383     * @param name attribute name
384     * @return attribute value for the given attribute
385     * @since 12510
386     */
387    protected static String getAttribute(Node node, String name) {
388        return node.getAttributes().getNamedItem(name).getNodeValue();
389    }
390
391    /**
392     * DOM document parser.
393     * @param <R> resulting type
394     * @since 12510
395     */
396    @FunctionalInterface
397    protected interface DomParser<R> {
398        /**
399         * Parses a given DOM document.
400         * @param doc DOM document
401         * @return parsed data
402         * @throws XmlParsingException if an XML parsing error occurs
403         */
404        R parse(Document doc) throws XmlParsingException;
405    }
406
407    /**
408     * Fetches generic data from the DOM document resulting an API call.
409     * @param api the OSM API call
410     * @param subtask the subtask translated message
411     * @param parser the parser converting the DOM document (OSM API result)
412     * @param <T> data type
413     * @param monitor The progress monitor
414     * @param reason The reason to show on console. Can be {@code null} if no reason is given
415     * @return The converted data
416     * @throws OsmTransferException if something goes wrong
417     * @since 12510
418     */
419    public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason)
420            throws OsmTransferException {
421        try {
422            monitor.beginTask("");
423            monitor.indeterminateSubTask(subtask);
424            try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) {
425                return parser.parse(XmlUtils.parseSafeDOM(in));
426            }
427        } catch (OsmTransferException e) {
428            throw e;
429        } catch (IOException | ParserConfigurationException | SAXException e) {
430            throw new OsmTransferException(e);
431        } finally {
432            monitor.finishTask();
433        }
434    }
435}