Coverage for netbox_data_import/identity.py: 100.00%

1# SPDX-License-Identifier: Apache-2.0 

2# SPDX-FileCopyrightText: 2026 Marcin Zieba <marcinpsk@gmail.com> 

3"""The one name identity: Python builds keys with it, and PostgreSQL compares stored names with it. 

4 

5Both sides map one explicit whitespace set to a space, collapse each run of spaces to one, trim the 

6ends, and then apply the full Unicode uppercase mapping. PostgreSQL uppercases under NetBox's 

7`natural_sort` ICU collation, whose locale is the root locale, so no language tailors the mapping. 

8No side applies NFC or NFKC. `test_identity` runs every storable code point through both sides. 

9A Source Adapter imports this module, so Django loads only inside the database helpers. 

10""" 

11 

12from __future__ import annotations 

13 

14import re 

15from collections.abc import Iterable 

16 

17# U+0009-000D, U+001C-0020, U+0085, U+00A0, U+1680, U+2000-200A, U+2028, U+2029, U+202F, U+205F, U+3000. 

18WHITESPACE = "".join( 

19 map( 

20 chr, 

21 ( 

22 *range(0x09, 0x0E), 

23 *range(0x1C, 0x21), 

24 0x85, 

25 0xA0, 

26 0x1680, 

27 *range(0x2000, 0x200B), 

28 0x2028, 

29 0x2029, 

30 0x202F, 

31 0x205F, 

32 0x3000, 

33 ), 

34 ) 

35) 

36# NetBox creates this collation with the ICU root locale, `und-u-kn-true`, for its own name columns. 

37IDENTITY_COLLATION = "natural_sort" 

38CANONICAL_NAME = "_ndi_canonical_name" 

39# A bracket expression of literal characters, which Python and PostgreSQL read the same way under any collation. 

40WHITESPACE_RUN = f"[{WHITESPACE}]+" 

41 

42_WHITESPACE_RUN = re.compile(WHITESPACE_RUN) 

43 

44 

45def identity_text(value: str) -> str: 

46 """Return the comparison key of one name.""" 

47 return _WHITESPACE_RUN.sub(" ", value).strip(" ").upper() 

48 

49 

50def identity_expression(expression): 

51 """Return the ORM expression that computes `identity_text` of one text value inside the database.""" 

52 from django.db.models import Func, TextField, Value 

53 from django.db.models.functions import Cast, Collate, Upper 

54 

55 # The bytewise collation keeps the input column's own collation out of every step before UPPER. 

56 text = Collate(Cast(expression, TextField()), "C") 

57 spaced = Func( 

58 text, Value(WHITESPACE_RUN), Value(" "), Value("g"), function="REGEXP_REPLACE", output_field=TextField() 

59 ) 

60 trimmed = Func(spaced, Value(" "), function="BTRIM", output_field=TextField()) 

61 return Upper(Collate(trimmed, IDENTITY_COLLATION)) 

62 

63 

64def identity_in(field: str, keys: Iterable[str]): 

65 """Return a filter that keeps the rows whose *field* has one of these identity keys.""" 

66 from django.db.models import F 

67 from django.db.models.lookups import In 

68 

69 return In(identity_expression(F(field)), sorted(set(keys))) 

70 

71 

72def with_name_identity(queryset): 

73 """Annotate each row with the identity of its name, under `CANONICAL_NAME`.""" 

74 from django.db.models import F 

75 

76 return queryset.annotate(**{CANONICAL_NAME: identity_expression(F("name"))}) 

77 

78 

79def matching_search(queryset, search: str): 

80 """Keep the named rows whose identity contains the identity of *search*; a blank search keeps every row.""" 

81 from django.db.models import F 

82 from django.db.models.lookups import Contains 

83 

84 wanted = identity_text(search) 

85 if not wanted: 

86 return queryset 

87 return queryset.filter(Contains(identity_expression(F("name")), wanted)) 

88 

89 

90__all__ = ( 

91 "CANONICAL_NAME", 

92 "IDENTITY_COLLATION", 

93 "WHITESPACE", 

94 "WHITESPACE_RUN", 

95 "identity_expression", 

96 "identity_in", 

97 "identity_text", 

98 "matching_search", 

99 "with_name_identity", 

100)