Resolve redirects when testing URLs for equality

The URL DOIs resolve to can move around, with redirects pointing to the
new location. To make the tests more robust, only fail if the URLs
differ after redirections.

See also
https://www.crossref.org/blog/urls-and-dois-a-complicated-relationship/

Author: hseg

tests/test_doi.py

@@ -3,13 +3,35 @@
 import os
 from pkg_resources import parse_version
 
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+from urllib.parse import urlparse, urlunparse
+
 import pytest
 
 from doi import (
     validate_doi, find_doi_in_text, __version__, pdf_to_doi,
     get_real_url_from_doi
 )
 
+def simplify_url(u):
+    return urlparse(u)._replace(query='', fragment='')
+
+def resolve_redirects(u):
+    u = urlunparse(urlparse(u)._replace(scheme='https'))
+    req = Request(u, headers={'User-Agent': 'Mozilla/5.0'})
+    try:
+        with urlopen(req) as r:
+            return simplify_url(r.url)
+    except HTTPError as e:
+        print(e)
+
+def normalize_eq(u, v):
+    return (u == v
+        or simplify_url(u) == simplify_url(v)
+        or resolve_redirects(u) == resolve_redirects(v)
+        )
+
 
 def test_valid_version() -> None:
     """Check that the package defines a valid __version__"""
@@ -32,7 +54,7 @@ def test_validate_doi() -> None:
             'https://linkinghub.elsevier.com/retrieve/pii/S0009261497040141'),
     ]
     for doi, url in data:
-        assert url == validate_doi(doi)
+        assert normalize_eq(url, validate_doi(doi))
 
     for doi in ['', 'asdf']:
         try:
@@ -49,7 +71,7 @@ def test_get_real_url_from_doi() -> None:
          'article/abs/pii/S0009261497040141'),
     ]
     for doi, url in data:
-        assert url == get_real_url_from_doi(doi)
+        assert normalize_eq(url, get_real_url_from_doi(doi))
 
 
 def test_find_doi_in_line() -> None: