From f61b04df898cb7776606a5d1e72094d4cf199969 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Wed, 16 Sep 2020 21:05:59 +0200
Subject: [PATCH 01/19] Add code that guesses a different capitalization in
 case of ambiguity. Fixes #161.

---
 quantulum3/classifier.py | 80 +++++++++++++++++++++++++++-------------
 1 file changed, 54 insertions(+), 26 deletions(-)

diff --git a/quantulum3/classifier.py b/quantulum3/classifier.py
index 80a41ab..bb5be1a 100644
--- a/quantulum3/classifier.py
+++ b/quantulum3/classifier.py
@@ -6,9 +6,10 @@
 # Standard library
 import json
 import logging
-import pkg_resources
-import os
 import multiprocessing
+import os
+
+import pkg_resources
 
 # Semi-dependencies
 try:
@@ -27,9 +28,9 @@
     wikipedia = None
 
 # Quantulum
-from . import load
+from . import language, load
 from .load import cached
-from . import language
+
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -109,7 +110,6 @@ def _clean_text_lang(lang):
 def train_classifier(
     parameters=None, ngram_range=(1, 1), store=True, lang="en_US", n_jobs=None
 ):
-
     """
     Train the intent classifier
     TODO auto invoke if sklearn version is new or first install or sth
@@ -245,7 +245,35 @@ def disambiguate_unit(unit, text, lang="en_US"):
     """
     Resolve ambiguity between units with same names, symbols or abbreviations.
     """
+    new_unit = disambiguate_unit_by_score(unit, text, lang)
+    if len(new_unit) == 1:
+        return next(iter(new_unit))
+
+    try:
+        # Instead of picking a random one now, we first change the
+        # capitalization of the unit and see if we can improve.
+        unit_changed = unit[:-1] + unit[-1].swapcase()
+        text_changed = text.replace(unit, unit_changed)
+
+        new_unit_changed = disambiguate_unit_by_score(unit_changed, text_changed, lang)
+        if len(new_unit_changed) == 1:
+            return next(iter(new_unit_changed))
+
+        if 0 < len(new_unit_changed) < len(new_unit):
+            # See if we have improved, otherwise we stick with the old new_unit.
+            new_unit = new_unit_changed
+
+    except KeyError:
+        pass  # Attempt failed, we just pick a random from new_unit now.
+
+    _LOGGER.warning(
+        "Could not resolve ambiguous units: '{}'. For unit '{}' in text '{}'. "
+        "Taking a random.".format(", ".join(str(u) for u in new_unit), unit, text)
+    )
+    return next(iter(new_unit))
+
 
+def disambiguate_unit_by_score(unit, text, lang):
     new_unit = (
         load.units(lang).symbols.get(unit)
         or load.units(lang).surfaces.get(unit)
@@ -254,25 +282,25 @@ def disambiguate_unit(unit, text, lang="en_US"):
     )
     if not new_unit:
         raise KeyError('Could not find unit "%s" from "%s"' % (unit, text))
+    if len(new_unit) == 1:
+        return new_unit
 
-    if len(new_unit) > 1:
-        transformed = classifier(lang).tfidf_model.transform([clean_text(text, lang)])
-        scores = classifier(lang).classifier.predict_proba(transformed).tolist()[0]
-        scores = zip(scores, classifier(lang).target_names)
-
-        # Filter for possible names
-        names = [i.name for i in new_unit]
-        scores = [i for i in scores if i[1] in names]
-
-        # Sort by rank
-        scores = sorted(scores, key=lambda x: x[0], reverse=True)
-        try:
-            final = load.units(lang).names[scores[0][1]]
-            _LOGGER.debug('\tAmbiguity resolved for "%s" (%s)' % (unit, scores))
-        except IndexError:
-            _LOGGER.debug('\tAmbiguity not resolved for "%s"' % unit)
-            final = next(iter(new_unit))
-    else:
-        final = next(iter(new_unit))
-
-    return final
+    # Start scoring
+    transformed = classifier(lang).tfidf_model.transform(
+        [clean_text(text, lang)]
+    )
+    scores = classifier(lang).classifier.predict_proba(transformed).tolist()[0]
+    scores = zip(scores, classifier(lang).target_names)
+
+    # Filter for possible names
+    names = [i.name for i in new_unit]
+    scores = [i for i in scores if i[1] in names]
+
+    # Sort by rank
+    scores = sorted(scores, key=lambda x: x[0], reverse=True)
+    try:
+        return [load.units(lang).names[scores[0][1]]]
+        _LOGGER.debug('\tAmbiguity resolved for "%s" (%s)' % (unit, scores))
+    except IndexError:
+        _LOGGER.debug('\tAmbiguity not resolved for "%s"' % unit)
+        return new_unit

From e0b05965d3d91df06c7c9291ce1398f459d5ff58 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Wed, 16 Sep 2020 21:38:49 +0200
Subject: [PATCH 02/19] Fixed some unreachable log line.

---
 quantulum3/classifier.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/quantulum3/classifier.py b/quantulum3/classifier.py
index bb5be1a..912ec2b 100644
--- a/quantulum3/classifier.py
+++ b/quantulum3/classifier.py
@@ -299,8 +299,8 @@ def disambiguate_unit_by_score(unit, text, lang):
     # Sort by rank
     scores = sorted(scores, key=lambda x: x[0], reverse=True)
     try:
-        return [load.units(lang).names[scores[0][1]]]
         _LOGGER.debug('\tAmbiguity resolved for "%s" (%s)' % (unit, scores))
+        return [load.units(lang).names[scores[0][1]]]
     except IndexError:
         _LOGGER.debug('\tAmbiguity not resolved for "%s"' % unit)
         return new_unit

From 47fc9ffd22d122cb84a3356da954cc873f71b0f7 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Wed, 16 Sep 2020 22:37:30 +0200
Subject: [PATCH 03/19] Slightly clearer what throws what now.

---
 quantulum3/classifier.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/quantulum3/classifier.py b/quantulum3/classifier.py
index 912ec2b..fa112e3 100644
--- a/quantulum3/classifier.py
+++ b/quantulum3/classifier.py
@@ -299,8 +299,9 @@ def disambiguate_unit_by_score(unit, text, lang):
     # Sort by rank
     scores = sorted(scores, key=lambda x: x[0], reverse=True)
     try:
+        new_unit = [load.units(lang).names[scores[0][1]]]
         _LOGGER.debug('\tAmbiguity resolved for "%s" (%s)' % (unit, scores))
-        return [load.units(lang).names[scores[0][1]]]
+        return new_unit
     except IndexError:
         _LOGGER.debug('\tAmbiguity not resolved for "%s"' % unit)
         return new_unit

From 368435b328e18318952aea3b4b91cf024e2fcaef Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Wed, 16 Sep 2020 23:37:53 +0200
Subject: [PATCH 04/19] Move capitalization logic one layer up.

---
 quantulum3/classifier.py    | 42 ++++---------------
 quantulum3/disambiguate.py  | 82 ++++++++++++++++++++++++++++---------
 quantulum3/no_classifier.py | 18 ++++++++
 3 files changed, 87 insertions(+), 55 deletions(-)

diff --git a/quantulum3/classifier.py b/quantulum3/classifier.py
index fa112e3..6a69a4a 100644
--- a/quantulum3/classifier.py
+++ b/quantulum3/classifier.py
@@ -11,6 +11,10 @@
 
 import pkg_resources
 
+# Quantulum
+from . import language, load
+from .load import cached
+
 # Semi-dependencies
 try:
     from sklearn.linear_model import SGDClassifier
@@ -27,10 +31,6 @@
 except ImportError:
     wikipedia = None
 
-# Quantulum
-from . import language, load
-from .load import cached
-
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -241,39 +241,11 @@ def disambiguate_entity(key, text, lang="en_US"):
 
 
 ###############################################################################
-def disambiguate_unit(unit, text, lang="en_US"):
-    """
-    Resolve ambiguity between units with same names, symbols or abbreviations.
-    """
-    new_unit = disambiguate_unit_by_score(unit, text, lang)
-    if len(new_unit) == 1:
-        return next(iter(new_unit))
-
-    try:
-        # Instead of picking a random one now, we first change the
-        # capitalization of the unit and see if we can improve.
-        unit_changed = unit[:-1] + unit[-1].swapcase()
-        text_changed = text.replace(unit, unit_changed)
-
-        new_unit_changed = disambiguate_unit_by_score(unit_changed, text_changed, lang)
-        if len(new_unit_changed) == 1:
-            return next(iter(new_unit_changed))
-
-        if 0 < len(new_unit_changed) < len(new_unit):
-            # See if we have improved, otherwise we stick with the old new_unit.
-            new_unit = new_unit_changed
-
-    except KeyError:
-        pass  # Attempt failed, we just pick a random from new_unit now.
-
-    _LOGGER.warning(
-        "Could not resolve ambiguous units: '{}'. For unit '{}' in text '{}'. "
-        "Taking a random.".format(", ".join(str(u) for u in new_unit), unit, text)
-    )
-    return next(iter(new_unit))
 
 
-def disambiguate_unit_by_score(unit, text, lang):
+def attempt_disambiguate_unit(unit, text, lang):
+    """Resolve ambiguity between units with same names, symbols or abbreviations.
+    Returns list of possibilities"""
     new_unit = (
         load.units(lang).symbols.get(unit)
         or load.units(lang).surfaces.get(unit)
diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index f81862d..4f2bcf5 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -3,42 +3,84 @@
 :mod:`Quantulum` disambiguation functions.
 """
 
+import logging
+
 # Quantulum
 from . import classifier as clf
-from . import no_classifier as no_clf
 from . import load
+from . import no_classifier as no_clf
 
+_LOGGER = logging.getLogger(__name__)
 
 ###############################################################################
+
+
 def disambiguate_unit(unit_surface, text, lang="en_US"):
     """
     Resolve ambiguity between units with same names, symbols or abbreviations.
     :returns (str) unit name of the resolved unit
     """
-    if clf.USE_CLF:
-        base = clf.disambiguate_unit(unit_surface, text, lang).name
-    else:
-        base = (
-            load.units(lang).symbols[unit_surface]
-            or load.units(lang).surfaces[unit_surface]
-            or load.units(lang).surfaces_lower[unit_surface.lower()]
-            or load.units(lang).symbols_lower[unit_surface.lower()]
-        )
-
-        if len(base) > 1:
-            base = no_clf.disambiguate_no_classifier(base, text, lang)
-        elif len(base) == 1:
-            base = next(iter(base))
-
-        if base:
-            base = base.name
+    units = attempt_disambiguate_unit(unit_surface, text, lang)
+    if units and len(units) == 1:
+        return next(iter(units)).name
+
+    if len(unit_surface) > 2:
+        # We will lower case everything except the first letter and see if
+        # there is a better match.
+        unit_changed = unit_surface[0] + unit_surface[1:].lower()
+        text_changed = text.replace(unit_surface, unit_changed)
+        new_units = attempt_disambiguate_unit(unit_changed, text_changed, lang)
+        units = get_a_better_one(units, new_units)
+        return resolve_ambiguity(units, unit_surface, text)
+
+    # Change the capitalization of the last letter to find a better match.
+    # The last better is sometimes cause of confusion, but the
+    # capitalization of the prefix is too important to alter.
+    unit_changed = unit_surface[:-1] + unit_surface[-1].swapcase()
+    text_changed = text.replace(unit_surface, unit_changed)
+    new_units = attempt_disambiguate_unit(unit_changed, text_changed, lang)
+    units = get_a_better_one(units, new_units)
+    return resolve_ambiguity(units, unit_surface, text)
+
+
+def attempt_disambiguate_unit(unit_surface, text, lang):
+    """Returns list of possibilities"""
+    try:
+        if clf.USE_CLF:
+            return clf.attempt_disambiguate_unit(unit_surface, text, lang)
         else:
-            base = "unk"
+            return no_clf.attempt_disambiguate_no_classifier(unit_surface, text, lang)
+    except KeyError:
+        return None
+
 
-    return base
+def get_a_better_one(old, new):
+    """Decide if we pick new over old, considering them being None, and 
+    preferring the smaller one."""
+    if not new:
+        return old
+    if not old:
+        return new
+    if len(new) < len(old):
+        return new
+    return old
+
+
+def resolve_ambiguity(units, unit, text):
+    if not units:
+        raise KeyError('Could not find unit "%s" from "%s"' % (unit, text))
+    if len(units) == 1:
+        return next(iter(units)).name
+    _LOGGER.warning(
+        "Could not resolve ambiguous units: '{}'. For unit '{}' in text '{}'. "
+        "Taking a random.".format(", ".join(str(u) for u in units), unit, text)
+    )
+    return next(iter(units)).name
 
 
 ###############################################################################
+
+
 def disambiguate_entity(key, text, lang="en_US"):
     """
     Resolve ambiguity between entities with same dimensionality.
diff --git a/quantulum3/no_classifier.py b/quantulum3/no_classifier.py
index 8b36cec..cd9b51a 100644
--- a/quantulum3/no_classifier.py
+++ b/quantulum3/no_classifier.py
@@ -34,3 +34,21 @@ def disambiguate_no_classifier(entities, text, lang="en_US"):
         if relative > max_relative or (relative == max_relative and count > max_count):
             max_entity, max_count, max_relative = entity, count, relative
     return max_entity
+
+
+def attempt_disambiguate_no_classifier(unit_surface, text, lang):
+    """Returns list of possibilities"""
+    base = (
+        load.units(lang).symbols[unit_surface]
+        or load.units(lang).surfaces[unit_surface]
+        or load.units(lang).surfaces_lower[unit_surface.lower()]
+        or load.units(lang).symbols_lower[unit_surface.lower()]
+    )
+    if not base:
+        raise KeyError('Could not find unit "%s" from "%s"' % (unit_surface, text))
+    if len(base) > 1:
+        possible_base = disambiguate_no_classifier(base, text, lang)
+    if not possible_base:
+        return base
+    else:
+        return [possible_base]

From 94a6afdc02efec3eca657f905f93e76fb8c41078 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Wed, 16 Sep 2020 23:58:00 +0200
Subject: [PATCH 05/19] Have determistic output when multiple things match.
 Addresses #163.

---
 quantulum3/disambiguate.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index 4f2bcf5..19a61dd 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -24,18 +24,15 @@ def disambiguate_unit(unit_surface, text, lang="en_US"):
     if units and len(units) == 1:
         return next(iter(units)).name
 
-    if len(unit_surface) > 2:
-        # We will lower case everything except the first letter and see if
-        # there is a better match.
-        unit_changed = unit_surface[0] + unit_surface[1:].lower()
-        text_changed = text.replace(unit_surface, unit_changed)
-        new_units = attempt_disambiguate_unit(unit_changed, text_changed, lang)
-        units = get_a_better_one(units, new_units)
-        return resolve_ambiguity(units, unit_surface, text)
-
     # Change the capitalization of the last letter to find a better match.
-    # The last better is sometimes cause of confusion, but the
+    # Capitalization is sometimes cause of confusion, but the
     # capitalization of the prefix is too important to alter.
+
+    # We don't change capitalization for units longer than 2.
+    # Than capitalization would not be a reason for problems.
+    if len(unit_surface) > 2:
+        return resolve_ambiguity(units, units, text)
+
     unit_changed = unit_surface[:-1] + unit_surface[-1].swapcase()
     text_changed = text.replace(unit_surface, unit_changed)
     new_units = attempt_disambiguate_unit(unit_changed, text_changed, lang)
@@ -55,7 +52,7 @@ def attempt_disambiguate_unit(unit_surface, text, lang):
 
 
 def get_a_better_one(old, new):
-    """Decide if we pick new over old, considering them being None, and 
+    """Decide if we pick new over old, considering them being None, and
     preferring the smaller one."""
     if not new:
         return old
@@ -75,7 +72,8 @@ def resolve_ambiguity(units, unit, text):
         "Could not resolve ambiguous units: '{}'. For unit '{}' in text '{}'. "
         "Taking a random.".format(", ".join(str(u) for u in units), unit, text)
     )
-    return next(iter(units)).name
+    # Deterministically getting something out of units.
+    return next(iter(sorted(units))).name
 
 
 ###############################################################################

From 8095df28ab0dcecbad6dd53cbaa35c045e848d0e Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Thu, 17 Sep 2020 00:00:47 +0200
Subject: [PATCH 06/19] No classifier does not raise KeyError but resorts to
 'unk' instead.

---
 quantulum3/disambiguate.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index 19a61dd..f06628c 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -65,7 +65,10 @@ def get_a_better_one(old, new):
 
 def resolve_ambiguity(units, unit, text):
     if not units:
-        raise KeyError('Could not find unit "%s" from "%s"' % (unit, text))
+        if clf.USE_CLF:
+            raise KeyError('Could not find unit "%s" from "%s"' % (unit, text))
+        else:
+            return "unk"
     if len(units) == 1:
         return next(iter(units)).name
     _LOGGER.warning(

From d11c1cd98d35b6249f0895a0824fa3661ce5989a Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Thu, 17 Sep 2020 00:05:19 +0200
Subject: [PATCH 07/19] Small reference error bug.

---
 quantulum3/no_classifier.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/quantulum3/no_classifier.py b/quantulum3/no_classifier.py
index cd9b51a..92d5c95 100644
--- a/quantulum3/no_classifier.py
+++ b/quantulum3/no_classifier.py
@@ -48,7 +48,6 @@ def attempt_disambiguate_no_classifier(unit_surface, text, lang):
         raise KeyError('Could not find unit "%s" from "%s"' % (unit_surface, text))
     if len(base) > 1:
         possible_base = disambiguate_no_classifier(base, text, lang)
-    if not possible_base:
-        return base
-    else:
-        return [possible_base]
+        if possible_base:
+            return [possible_base]
+    return base

From 17237f7c51908d180f76c1c48101984f970183b8 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Thu, 17 Sep 2020 00:17:55 +0200
Subject: [PATCH 08/19] Removing notion of random.

---
 quantulum3/disambiguate.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index f06628c..16a0e44 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -72,8 +72,9 @@ def resolve_ambiguity(units, unit, text):
     if len(units) == 1:
         return next(iter(units)).name
     _LOGGER.warning(
-        "Could not resolve ambiguous units: '{}'. For unit '{}' in text '{}'. "
-        "Taking a random.".format(", ".join(str(u) for u in units), unit, text)
+        "Could not resolve ambiguous units: '{}'. For unit '{}' in text '{}'. ".format(
+            ", ".join(str(u) for u in units), unit, text
+        )
     )
     # Deterministically getting something out of units.
     return next(iter(sorted(units))).name

From 7ad9ecbc00913964868e4a8fba0f1d2db8e9add7 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Thu, 17 Sep 2020 00:27:56 +0200
Subject: [PATCH 09/19] Sorting not supported on 'Unit'.

---
 quantulum3/disambiguate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index 16a0e44..50e737d 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -77,7 +77,7 @@ def resolve_ambiguity(units, unit, text):
         )
     )
     # Deterministically getting something out of units.
-    return next(iter(sorted(units))).name
+    return next(iter(sorted(u.name for u in units)))
 
 
 ###############################################################################

From 22e163c3773eb3f6e30333c9c3d2867b15c9b892 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Thu, 17 Sep 2020 00:50:43 +0200
Subject: [PATCH 10/19] Consider capitalization for units longer than 2
 letters.

---
 quantulum3/disambiguate.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index 50e737d..2395335 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -28,9 +28,13 @@ def disambiguate_unit(unit_surface, text, lang="en_US"):
     # Capitalization is sometimes cause of confusion, but the
     # capitalization of the prefix is too important to alter.
 
-    # We don't change capitalization for units longer than 2.
-    # Than capitalization would not be a reason for problems.
+    # If the unit is longer than two prefixes, we set everything to lower
+    # except the first letter.
     if len(unit_surface) > 2:
+        unit_changed = unit_surface[0] + unit_surface[1:].lower()
+        text_changed = text.replace(unit_surface, unit_changed)
+        new_units = attempt_disambiguate_unit(unit_changed, text_changed, lang)
+        units = get_a_better_one(units, new_units)
         return resolve_ambiguity(units, units, text)
 
     unit_changed = unit_surface[:-1] + unit_surface[-1].swapcase()

From 336fae5b6297fb53d532efff8919a1ed3179694b Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Thu, 17 Sep 2020 00:57:14 +0200
Subject: [PATCH 11/19] Changed some conditions that altered capitalization
 work around.

---
 quantulum3/disambiguate.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index 2395335..49d8985 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -32,10 +32,16 @@ def disambiguate_unit(unit_surface, text, lang="en_US"):
     # except the first letter.
     if len(unit_surface) > 2:
         unit_changed = unit_surface[0] + unit_surface[1:].lower()
+        if unit_changed == unit_surface:
+            return resolve_ambiguity(units, unit_surface, text)
         text_changed = text.replace(unit_surface, unit_changed)
         new_units = attempt_disambiguate_unit(unit_changed, text_changed, lang)
         units = get_a_better_one(units, new_units)
-        return resolve_ambiguity(units, units, text)
+        return resolve_ambiguity(units, unit_surface, text)
+
+    if unit_surface[0] not in load.METRIC_PREFIXES.keys():
+        # Only apply next work around if the first letter is a SI-prefix
+        return resolve_ambiguity(units, unit_surface, text)
 
     unit_changed = unit_surface[:-1] + unit_surface[-1].swapcase()
     text_changed = text.replace(unit_surface, unit_changed)

From 5cd0192e2256566fdac985916300c5d43574fe42 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Fri, 25 Sep 2020 11:04:33 +0200
Subject: [PATCH 12/19] Attempt in trying to add a test.

---
 quantulum3/tests/test_classifier.py | 42 +++++++++++++++++++----------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/quantulum3/tests/test_classifier.py b/quantulum3/tests/test_classifier.py
index b87beaf..5ec6654 100644
--- a/quantulum3/tests/test_classifier.py
+++ b/quantulum3/tests/test_classifier.py
@@ -6,28 +6,24 @@
 
 from __future__ import division
 
+import json
 # Standard library
 import os
-import json
-import urllib.request
 import unittest
-
-# Quantulum
-from .. import load
-from .. import parser as p
-from .. import classifier as clf
-from .. import language
-from .test_setup import (
-    load_expand_tests,
-    load_quantity_tests,
-    multilang,
-    add_type_equalities,
-)
+import urllib.request
 
 # Dependencies
 import joblib
 import wikipedia
 
+# Quantulum
+from .. import classifier as clf
+from .. import language, load
+from .. import parser as p
+from ..classes import Entity, Quantity, Unit
+from .test_setup import (add_type_equalities, load_expand_tests,
+                         load_quantity_tests, multilang)
+
 COLOR1 = "\033[94m%s\033[0m"
 COLOR2 = "\033[91m%s\033[0m"
 TOPDIR = os.path.dirname(__file__) or "."
@@ -155,6 +151,24 @@ def test_wikipedia_pages(self, lang):
             self.fail("Problematic pages:\n{}".format("\n".join(str(e) for e in err)))
 
 
+class ClassifierTestEdgeCases(unittest.TestCase):
+    """Test suite that tests certain specifically designed edge cases to confuse the classifier."""
+
+    def test_wrong_capitalization(self):
+        parsed_unit = p.parse('1nw')
+        unit = Quantity(
+            value=1,
+            unit=Unit(
+                name="nanowatt",
+                entity=Entity("power"),
+                dimensions=[
+                    {"base": "nanowatt", "power": 1, "surface": "nW"}
+                ]
+            )
+        )
+        self.assertEqual(parsed_unit, unit)
+
+
 ###############################################################################
 if __name__ == "__main__":  # pragma: no cover
 

From ec38051799b14824820f3dc339dcb48dc52c88e5 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Sat, 24 Oct 2020 21:57:47 +0200
Subject: [PATCH 13/19] Solve merge conflict

---
 quantulum3/classifier.py            | 4 +---
 quantulum3/tests/test_classifier.py | 8 +++-----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/quantulum3/classifier.py b/quantulum3/classifier.py
index 2d55e50..a9119fb 100644
--- a/quantulum3/classifier.py
+++ b/quantulum3/classifier.py
@@ -256,9 +256,7 @@ def attempt_disambiguate_unit(unit, text, lang):
         return new_unit
 
     # Start scoring
-    transformed = classifier(lang).tfidf_model.transform(
-        [clean_text(text, lang)]
-    )
+    transformed = classifier(lang).tfidf_model.transform([clean_text(text, lang)])
     scores = classifier(lang).classifier.predict_proba(transformed).tolist()[0]
     scores = zip(scores, classifier(lang).target_names)
 
diff --git a/quantulum3/tests/test_classifier.py b/quantulum3/tests/test_classifier.py
index 093e27b..3aa7906 100644
--- a/quantulum3/tests/test_classifier.py
+++ b/quantulum3/tests/test_classifier.py
@@ -171,16 +171,14 @@ class ClassifierTestEdgeCases(unittest.TestCase):
     """Test suite that tests certain specifically designed edge cases to confuse the classifier."""
 
     def test_wrong_capitalization(self):
-        parsed_unit = p.parse('1nw')
+        parsed_unit = p.parse("1nw")
         unit = Quantity(
             value=1,
             unit=Unit(
                 name="nanowatt",
                 entity=Entity("power"),
-                dimensions=[
-                    {"base": "nanowatt", "power": 1, "surface": "nW"}
-                ]
-            )
+                dimensions=[{"base": "nanowatt", "power": 1, "surface": "nW"}],
+            ),
         )
         self.assertEqual(parsed_unit, unit)
 

From 4d70c3d257f6b47fe23e9f4aaf6f85ea3de0f5ca Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Tue, 27 Oct 2020 13:48:22 +0100
Subject: [PATCH 14/19] Add test case.

---
 quantulum3/_lang/en_US/tests/quantities.json | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/quantulum3/_lang/en_US/tests/quantities.json b/quantulum3/_lang/en_US/tests/quantities.json
index 981a21c..ab83a37 100644
--- a/quantulum3/_lang/en_US/tests/quantities.json
+++ b/quantulum3/_lang/en_US/tests/quantities.json
@@ -1373,5 +1373,15 @@
           "surface": "three million, two hundred & forty"
         }
       ]
+  },
+  {
+    "req": "The battery has 2nw.",
+    "res": [
+        {
+            "value": 2,
+            "unit": "nanowatt",
+            "surface": "2nW"
+        }
+    ]
   }
 ]

From d387e797fe8822bad1159dd97434d29701cf850e Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Tue, 27 Oct 2020 14:03:12 +0100
Subject: [PATCH 15/19] Added test case.

---
 quantulum3/_lang/en_US/tests/quantities.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/quantulum3/_lang/en_US/tests/quantities.json b/quantulum3/_lang/en_US/tests/quantities.json
index ab83a37..7575acd 100644
--- a/quantulum3/_lang/en_US/tests/quantities.json
+++ b/quantulum3/_lang/en_US/tests/quantities.json
@@ -1380,7 +1380,7 @@
         {
             "value": 2,
             "unit": "nanowatt",
-            "surface": "2nW"
+            "surface": "2nw"
         }
     ]
   }

From 633a4b2291a51b4ac65a86bce2f85fb1e870ca47 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Tue, 27 Oct 2020 14:10:13 +0100
Subject: [PATCH 16/19] Remove old test case.

---
 quantulum3/tests/test_classifier.py | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/quantulum3/tests/test_classifier.py b/quantulum3/tests/test_classifier.py
index 3aa7906..607728b 100644
--- a/quantulum3/tests/test_classifier.py
+++ b/quantulum3/tests/test_classifier.py
@@ -167,22 +167,6 @@ def test_wikipedia_pages(self, lang):
             self.fail("Problematic pages:\n{}".format("\n".join(str(e) for e in err)))
 
 
-class ClassifierTestEdgeCases(unittest.TestCase):
-    """Test suite that tests certain specifically designed edge cases to confuse the classifier."""
-
-    def test_wrong_capitalization(self):
-        parsed_unit = p.parse("1nw")
-        unit = Quantity(
-            value=1,
-            unit=Unit(
-                name="nanowatt",
-                entity=Entity("power"),
-                dimensions=[{"base": "nanowatt", "power": 1, "surface": "nW"}],
-            ),
-        )
-        self.assertEqual(parsed_unit, unit)
-
-
 ###############################################################################
 if __name__ == "__main__":  # pragma: no cover
     unittest.main()

From b941b158d15f69664f9c6e1571f45f33fa9731a0 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Tue, 27 Oct 2020 14:23:37 +0100
Subject: [PATCH 17/19] Solve strange edge case.

---
 quantulum3/disambiguate.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index abb6948..cd3f684 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -38,7 +38,7 @@ def disambiguate_unit(unit_surface, text, lang="en_US"):
         units = get_a_better_one(units, new_units)
         return resolve_ambiguity(units, unit_surface, text, lang)
 
-    if unit_surface[0] not in load.METRIC_PREFIXES.keys():
+    if not unit_surface or unit_surface[0] not in load.METRIC_PREFIXES.keys():
         # Only apply next work around if the first letter is a SI-prefix
         return resolve_ambiguity(units, unit_surface, text, lang)
 
@@ -74,10 +74,10 @@ def get_a_better_one(old, new):
 
 def resolve_ambiguity(units, unit, text, lang):
     if not units:
-        if clf.USE_CLF:
+        if unit and clf.USE_CLF:
             raise KeyError('Could not find unit "%s" from "%s"' % (unit, text))
         else:
-            return load.units(lang).names.get("unk")
+            return 'unk'
     if len(units) == 1:
         return next(iter(units)).name
     _LOGGER.warning(

From 0e8cafa1e7cb24b9a7c5cac24264da90c462ffe3 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Tue, 27 Oct 2020 14:41:13 +0100
Subject: [PATCH 18/19] Pylint issues.

---
 quantulum3/disambiguate.py          | 10 +++++-----
 quantulum3/tests/test_classifier.py |  1 -
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index cd3f684..5fb5d40 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -32,21 +32,21 @@ def disambiguate_unit(unit_surface, text, lang="en_US"):
     if len(unit_surface) > 2:
         unit_changed = unit_surface[0] + unit_surface[1:].lower()
         if unit_changed == unit_surface:
-            return resolve_ambiguity(units, unit_surface, text, lang)
+            return resolve_ambiguity(units, unit_surface, text)
         text_changed = text.replace(unit_surface, unit_changed)
         new_units = attempt_disambiguate_unit(unit_changed, text_changed, lang)
         units = get_a_better_one(units, new_units)
-        return resolve_ambiguity(units, unit_surface, text, lang)
+        return resolve_ambiguity(units, unit_surface, text)
 
     if not unit_surface or unit_surface[0] not in load.METRIC_PREFIXES.keys():
         # Only apply next work around if the first letter is a SI-prefix
-        return resolve_ambiguity(units, unit_surface, text, lang)
+        return resolve_ambiguity(units, unit_surface, text)
 
     unit_changed = unit_surface[:-1] + unit_surface[-1].swapcase()
     text_changed = text.replace(unit_surface, unit_changed)
     new_units = attempt_disambiguate_unit(unit_changed, text_changed, lang)
     units = get_a_better_one(units, new_units)
-    return resolve_ambiguity(units, unit_surface, text, lang)
+    return resolve_ambiguity(units, unit_surface, text)
 
 
 def attempt_disambiguate_unit(unit_surface, text, lang):
@@ -72,7 +72,7 @@ def get_a_better_one(old, new):
     return old
 
 
-def resolve_ambiguity(units, unit, text, lang):
+def resolve_ambiguity(units, unit, text):
     if not units:
         if unit and clf.USE_CLF:
             raise KeyError('Could not find unit "%s" from "%s"' % (unit, text))
diff --git a/quantulum3/tests/test_classifier.py b/quantulum3/tests/test_classifier.py
index 607728b..1c2812b 100644
--- a/quantulum3/tests/test_classifier.py
+++ b/quantulum3/tests/test_classifier.py
@@ -17,7 +17,6 @@
 from .. import classifier as clf
 from .. import language, load
 from .. import parser as p
-from ..classes import Entity, Quantity, Unit
 from .test_setup import (
     add_type_equalities,
     load_error_tests,

From 378ef5fe61945f84a9d6897a63d3a1aca1b271a1 Mon Sep 17 00:00:00 2001
From: Hielke Walinga <hielkewalinga@gmail.com>
Date: Tue, 27 Oct 2020 15:03:07 +0100
Subject: [PATCH 19/19] Black reformat.

---
 quantulum3/disambiguate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/quantulum3/disambiguate.py b/quantulum3/disambiguate.py
index 5fb5d40..14050f2 100644
--- a/quantulum3/disambiguate.py
+++ b/quantulum3/disambiguate.py
@@ -77,7 +77,7 @@ def resolve_ambiguity(units, unit, text):
         if unit and clf.USE_CLF:
             raise KeyError('Could not find unit "%s" from "%s"' % (unit, text))
         else:
-            return 'unk'
+            return "unk"
     if len(units) == 1:
         return next(iter(units)).name
     _LOGGER.warning(