# HG changeset patch
# User Thomas De Schampheleire <thomas.de_schampheleire@nokia.com>
# Date 2018-02-14 09:12:17
# Node ID f91844b26269535f00e0770f8a533d9e2748c032
# Parent  28317fc212f4ffe125ff627a018ba4928c46c731

lib: fix detection of &#39; as issue reference

Commit 494c793cc160 changed HTML escaping to please HTML 4 email readers.
The HTML entity '&apos;' was replaced by '&#39;'.
Unfortunately, the pound character '#' is often used to mark issue
references, like 'bug #56'. While this depends on the issue patterns
actually configured, this pattern is so common that we cannot expect users
to set their issue_pat regular expressions such that '&#123;' is not
matched.

Instead, keep the original '&apos;' replacement at first in method html_escape,
but introduce a final step that just replaces '&apos;' with '&#39;'.

The order of replacement in urlify_text then changes from:
    html_escape (to HTML4)
    urlify_issues
to
    html_escape (to HTML5)
    urlify_issues
    make HTML5 more like HTML4

Test coverage show the problem case being solved.

diff --git a/kallithea/lib/helpers.py b/kallithea/lib/helpers.py
--- a/kallithea/lib/helpers.py
+++ b/kallithea/lib/helpers.py
@@ -85,7 +85,7 @@ def html_escape(s):
         .replace(">", "&gt;")
         .replace("<", "&lt;")
         .replace('"', "&quot;")
-        .replace("'", "&#39;") # some mail readers use HTML 4 and doesn't support &apos;
+        .replace("'", "&apos;") # Note: this is HTML5 not HTML4 and might not work in mails
         )
 
 def js(value):
@@ -1092,6 +1092,10 @@ def urlify_text(s, repo_name=None, link_
         # make href around everything that isn't a href already
         s = linkify_others(s, link_)
     s = s.replace('\r\n', '<br/>').replace('\n', '<br/>')
+    # Turn HTML5 into more valid HTML4 as required by some mail readers.
+    # (This is not done in one step in html_escape, because character codes like
+    # &#123; risk to be seen as an issue reference due to the presence of '#'.)
+    s = s.replace("&apos;", "&#39;")
     return literal(s)
 
 
diff --git a/kallithea/tests/other/test_libs.py b/kallithea/tests/other/test_libs.py
--- a/kallithea/tests/other/test_libs.py
+++ b/kallithea/tests/other/test_libs.py
@@ -381,8 +381,7 @@ class TestLibs(TestController):
        """and not be followed by * or alphanumerical <b>*characters*</b>.""",
        "-"),
       ("HTML escaping: <abc> 'single' \"double\" &pointer",
-       # problem: ' is encoded as &#39; which however is interpreted as #39 and expanded to a issue link
-       """HTML escaping: &lt;abc&gt; &<a class="issue-tracker-link" href="https://issues.example.com/repo_name/issue/39">#39</a>;single&<a class="issue-tracker-link" href="https://issues.example.com/repo_name/issue/39">#39</a>; &quot;double&quot; &amp;pointer""",
+       "HTML escaping: &lt;abc&gt; &#39;single&#39; &quot;double&quot; &amp;pointer",
        "-"),
       # tags are covered by test_tag_extractor
     ])
@@ -420,11 +419,10 @@ class TestLibs(TestController):
             'silly me, the URL does not contain {id}, BUG12345.', 'silly me, the URL does not contain {id}, <a class="issue-tracker-link" href="https://bar/repo_name/">BUG12345</a>.'),
         (r'(PR-\d+)', 'http://foo/{repo}/issue/{id}', '',
             'interesting issue #123, err PR-56', 'interesting issue #123, err <a class="issue-tracker-link" href="http://foo/repo_name/issue/PR-56">PR-56</a>'),
-        # problem: ' is encoded as &#39; which however is interpreted as #39 and expanded to a issue link
         (r'#(\d+)', 'http://foo/{repo}/issue/{id}', '#',
-            "some 'standard' text with apostrophes", 'some &<a class="issue-tracker-link" href="http://foo/repo_name/issue/39">#39</a>;standard&<a class="issue-tracker-link" href="http://foo/repo_name/issue/39">#39</a>; text with apostrophes'),
+            "some 'standard' text with apostrophes", 'some &#39;standard&#39; text with apostrophes'),
         (r'#(\d+)', 'http://foo/{repo}/issue/{id}', '#',
-            "some 'standard' issue #123", 'some &<a class="issue-tracker-link" href="http://foo/repo_name/issue/39">#39</a>;standard&<a class="issue-tracker-link" href="http://foo/repo_name/issue/39">#39</a>; issue <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a>'),
+            "some 'standard' issue #123", 'some &#39;standard&#39; issue <a class="issue-tracker-link" href="http://foo/repo_name/issue/123">#123</a>'),
     ])
     def test_urlify_issues(self, issue_pat, issue_server, issue_prefix, sample, expected):
         from kallithea.lib.helpers import urlify_text