Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
586a022
Added support for dbi struct parameters with explicit types
Jun 22, 2021
13f9107
Make the dataset_id fixture a session fixture
Jun 23, 2021
aea7bae
Make the dataset_id fixture a session fixture
Jun 23, 2021
6f26130
system test of the struct machinery
Jun 23, 2021
c386448
Verify that we can bind non-parameterized types
Jun 23, 2021
71c8614
Parse and remove type parameters from explcit types.
Jun 23, 2021
6f5b345
Document passing struct data.
Jun 23, 2021
e08f6f6
Merge remote-tracking branch 'origin/master' into riversnake-dbi-stru…
Jun 23, 2021
411c336
blacken
Jun 23, 2021
ef2b323
using match.groups() throws off pytypes, also fix some type hints.
Jun 23, 2021
654b108
🦉 Updates from OwlBot
gcf-owl-bot[bot] Jun 23, 2021
525b8fd
blacken
Jun 23, 2021
462f2eb
remove type hints -- maybe they broke docs?
Jun 23, 2021
904d2ce
merge upstream
Jun 23, 2021
a6393e6
Revert "remove type hints -- maybe they broke docs?"
Jun 23, 2021
e63e8b7
pin gcp-sphinx-docfx-yaml==0.2.0 so docfx doesn't fail.
Jun 23, 2021
2f2bdcd
Merge remote-tracking branch 'origin/master' into riversnake-dbi-stru…
Jun 24, 2021
91b0028
Review comments: examples, and guard against large number of fields
Jun 24, 2021
35555aa
🦉 Updates from OwlBot
gcf-owl-bot[bot] Jun 24, 2021
0d81b80
Merge remote-tracking branch 'origin/master' into riversnake-dbi-stru…
Jun 24, 2021
d3f959c
Merge branch 'riversnake-dbi-struct-types' of github.com:googleapis/p…
Jun 24, 2021
5554301
Factored some repeated code in handling complex parameters
Jun 25, 2021
8830113
Improved the error for dict (structish) parameter values without expl…
Jun 25, 2021
a72cafb
blacken
Jun 25, 2021
cba9697
removed repeated word
Jun 25, 2021
12bd941
Update google/cloud/bigquery/dbapi/_helpers.py
Jun 29, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
blacken
  • Loading branch information
Jim Fulton committed Jun 23, 2021
commit 411c336191ce1d15879dd8fc3fec3975df54c22c
82 changes: 45 additions & 37 deletions google/cloud/bigquery/dbapi/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,23 @@
_NUMERIC_SERVER_MIN = decimal.Decimal("-9.9999999999999999999999999999999999999E+28")
_NUMERIC_SERVER_MAX = decimal.Decimal("9.9999999999999999999999999999999999999E+28")

type_parameters_re = re.compile(r"""
type_parameters_re = re.compile(
r"""
\(
\s*[0-9]+\s*
(,
\s*[0-9]+\s*
)*
\)
""", re.VERBOSE)
""",
re.VERBOSE,
)


def _parameter_type(name, value, query_parameter_type=None, value_doc=""):
if query_parameter_type:
# Strip type parameters
query_parameter_type = type_parameters_re.sub('', query_parameter_type)
query_parameter_type = type_parameters_re.sub("", query_parameter_type)
try:
parameter_type = getattr(
enums.SqlParameterScalarTypes, query_parameter_type.upper()
Expand Down Expand Up @@ -134,35 +137,38 @@ def array_to_query_parameter(value, name=None, query_parameter_type=None):
\s*$
""",
re.IGNORECASE | re.VERBOSE,
).match
).match
parse_struct_field = re.compile(
r"""
(?:(\w+)\s+) # field name
([A-Z0-9<> ,()]+) # Field type
$""", re.VERBOSE | re.IGNORECASE).match
$""",
re.VERBOSE | re.IGNORECASE,
).match


def split_struct_fields(fields):
fields = fields.split(',')
fields = fields.split(",")
while fields:
field = fields.pop(0)
while fields and field.count('<') != field.count('>'):
field += ',' + fields.pop(0)
while fields and field.count("<") != field.count(">"):
field += "," + fields.pop(0)
yield field


def complex_query_parameter_type(name: str, type_: str, base: str):
type_ = type_.strip()
if '<' not in type_:
if "<" not in type_:
# Scalar

# Strip type parameters
type_ = type_parameters_re.sub('', type_).strip()
type_ = type_parameters_re.sub("", type_).strip()
try:
type_ = getattr(enums.SqlParameterScalarTypes, type_.upper())
except AttributeError:
raise exceptions.ProgrammingError(
f"Invalid scalar type, {type_}, in {base}")
f"Invalid scalar type, {type_}, in {base}"
)
if name:
type_ = type_.with_name(name)
return type_
Expand All @@ -173,21 +179,21 @@ def complex_query_parameter_type(name: str, type_: str, base: str):
tname, sub = m.groups()
tname = tname.upper()
sub = sub.strip()
if tname == 'ARRAY':
if tname == "ARRAY":
return query.ArrayQueryParameterType(
complex_query_parameter_type(None, sub, base),
name=name)
complex_query_parameter_type(None, sub, base), name=name
)
else:
fields = []
for field_string in split_struct_fields(sub):
field_string = field_string.strip()
m = parse_struct_field(field_string)
if not m:
raise exceptions.ProgrammingError(
f"Invalid struct field, {field_string}, in {base}")
f"Invalid struct field, {field_string}, in {base}"
)
field_name, field_type = m.groups()
fields.append(complex_query_parameter_type(
field_name, field_type, base))
fields.append(complex_query_parameter_type(field_name, field_type, base))

return query.StructQueryParameterType(*fields, name=name)

Expand All @@ -200,18 +206,18 @@ def complex_query_parameter(name, value, type_, base=None):
"""
type_ = type_.strip()
base = base or type_
if '>' not in type_:
if ">" not in type_:
# Scalar

# Strip type parameters
type_ = type_parameters_re.sub('', type_).strip()
type_ = type_parameters_re.sub("", type_).strip()
try:
type_ = getattr(enums.SqlParameterScalarTypes, type_.upper())._type
except AttributeError:
raise exceptions.ProgrammingError(
f"The given parameter type, {type_},"
f" for {name} is not a valid BigQuery scalar type, in {base}."
)
)

return query.ScalarQueryParameter(name, type_, value)

Expand All @@ -221,41 +227,43 @@ def complex_query_parameter(name, value, type_, base=None):
tname, sub = m.groups()
tname = tname.upper()
sub = sub.strip()
if tname == 'ARRAY':
if tname == "ARRAY":
if not array_like(value):
raise exceptions.ProgrammingError(
f"Array type with non-array-like value"
f" with type {type(value).__name__}")
f" with type {type(value).__name__}"
)
array_type = complex_query_parameter_type(name, sub, base)
if isinstance(array_type, query.ArrayQueryParameterType):
raise exceptions.ProgrammingError(f"Array can't contain an array in {base}")
return query.ArrayQueryParameter(
name,
array_type,
[complex_query_parameter(None, v, sub, base)
for v in value] if '<' in sub else value,
)
[complex_query_parameter(None, v, sub, base) for v in value]
if "<" in sub
else value,
)
else:
fields = []
if not isinstance(value, collections_abc.Mapping):
raise exceptions.ProgrammingError(
f"Non-mapping value for type {type_}")
raise exceptions.ProgrammingError(f"Non-mapping value for type {type_}")
value_keys = set(value)
for field_string in split_struct_fields(sub):
field_string = field_string.strip()
m = parse_struct_field(field_string)
if not m:
raise exceptions.ProgrammingError(
f"Invalid struct field, {field_string}, in {base or type_}")
f"Invalid struct field, {field_string}, in {base or type_}"
)
field_name, field_type = m.groups()
if field_name not in value:
raise exceptions.ProgrammingError(
f"No field value for {field_name} in {type_}")
f"No field value for {field_name} in {type_}"
)
value_keys.remove(field_name)
fields.append(
complex_query_parameter(
field_name, value[field_name], field_type, base)
)
complex_query_parameter(field_name, value[field_name], field_type, base)
)
if value_keys:
raise exceptions.ProgrammingError(f"Extra data keys for {type_}")

Expand All @@ -278,7 +286,7 @@ def to_query_parameters_list(parameters, parameter_types):
result = []

for value, type_ in zip(parameters, parameter_types):
if type_ is not None and '<' in type_:
if type_ is not None and "<" in type_:
param = complex_query_parameter(None, value, type_)
elif isinstance(value, collections_abc.Mapping):
raise NotImplementedError("STRUCT-like parameter values are not supported.")
Expand Down Expand Up @@ -309,21 +317,21 @@ def to_query_parameters_dict(parameters, query_parameter_types):

for name, value in parameters.items():
query_parameter_type = query_parameter_types.get(name)
if query_parameter_type is not None and '<' in query_parameter_type:
if query_parameter_type is not None and "<" in query_parameter_type:
param = complex_query_parameter(name, value, query_parameter_type)
elif isinstance(value, collections_abc.Mapping):
raise NotImplementedError(
"STRUCT-like parameter values are not supported "
"(parameter {}).".format(name)
)
)
elif array_like(value):
param = array_to_query_parameter(
value, name=name, query_parameter_type=query_parameter_type
)
)
else:
param = scalar_to_query_parameter(
value, name=name, query_parameter_type=query_parameter_type,
)
)

result.append(param)

Expand Down
6 changes: 4 additions & 2 deletions google/cloud/bigquery/dbapi/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -483,7 +483,8 @@ def _format_operation(operation, parameters):


def _extract_types(
operation, extra_type_sub=re.compile(
operation,
extra_type_sub=re.compile(
r"""
(%*) # Extra %s. We'll deal with these in the replacement code

Expand All @@ -507,7 +508,8 @@ def _extract_types(

s # End of replacement
""",
re.VERBOSE).sub
re.VERBOSE,
).sub,
):
"""Remove type information from parameter placeholders.

Expand Down
1 change: 1 addition & 0 deletions tests/system/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def dataset_id(bigquery_client):
yield dataset_id
bigquery_client.delete_dataset(dataset_id, delete_contents=True)


@pytest.fixture
def table_id(dataset_id):
return f"{dataset_id}.table_{helpers.temp_suffix()}"
30 changes: 15 additions & 15 deletions tests/system/test_structs.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,27 @@

from google.cloud.bigquery.dbapi import connect

person_type = ('struct<name string,'
' children array<struct<name string, bdate date>>>')
person_type_sized = ('struct<name string(22),'
' children array<struct<name string(22), bdate date>>>')
person_type = "struct<name string," " children array<struct<name string, bdate date>>>"
person_type_sized = (
"struct<name string(22)," " children array<struct<name string(22), bdate date>>>"
)

@pytest.mark.parametrize(
"person_type_decl", [person_type, person_type_sized]
)

@pytest.mark.parametrize("person_type_decl", [person_type, person_type_sized])
def test_structs(bigquery_client, dataset_id, person_type_decl, table_id):
conn = connect(bigquery_client)
cursor = conn.cursor()
cursor.execute(f"create table {table_id} (person {person_type_decl})")
data = dict(name='par',
children=[
dict(name='ch1', bdate=datetime.date(2021, 1, 1)),
dict(name='ch2', bdate=datetime.date(2021, 1, 2)),
])
data = dict(
name="par",
children=[
dict(name="ch1", bdate=datetime.date(2021, 1, 1)),
dict(name="ch2", bdate=datetime.date(2021, 1, 2)),
],
)
cursor.execute(
f"insert into {table_id} (person) values (%(v:{person_type})s)",
dict(v=data),
)
f"insert into {table_id} (person) values (%(v:{person_type})s)", dict(v=data),
)

cursor.execute(f"select * from {table_id}")
[[result]] = list(cursor)
Expand Down
Loading