Programmiererforum

CREATE TABLE family (
id INT,
data VARIANT
);

INSERT INTO family (id, data)
VALUES
(1, PARSE_JSON('{"name":"Alice","age":30}')),
(2, PARSE_JSON('[1,2,3,4,5]')),
(3, PARSE_JSON('42'));

SELECT
id,
try_variant_get(data, '$.name', 'STRING') AS name,
try_variant_get(data, '$.age', 'INT') AS age
FROM
family
WHERE
try_variant_get(data, '$.name', 'STRING') IS NOT NULL;

SparkSession spark = SparkSession.builder().master("local[*]").appName("VariantExample").getOrCreate();

StructType schema = new StructType()
.add("id", DataTypes.IntegerType)
.add("data", DataTypes.VariantType);

Dataset df = spark.createDataFrame(
Arrays.asList(
RowFactory.create(1, "{\"name\":\"Alice\",\"age\":30}"),
RowFactory.create(2, "[1,2,3,4,5]"),
RowFactory.create(3, "42")
),
schema
);

Dataset df_sel = df.select(
col("id"),
try_variant_get(col("data"), "$.name", "String").alias("name"),
try_variant_get(col("data"), "$.age", "Integer").alias("age")
).where("name IS NOT NULL");

df_sel.printSchema();
df_sel.show();

root
|-- id: integer (nullable = true)
|-- name: string (nullable = true)
|-- age: integer (nullable = true)

Exception in thread "main" java.lang.ClassCastException: class java.lang.String cannot be cast to class org.apache.spark.unsafe.types.VariantVal (java.lang.String is in module java.base of loader 'bootstrap'; org.apache.spark.unsafe.types.VariantVal is in unnamed module of loader 'app')
at org.apache.spark.sql.catalyst.expressions.variant.VariantGet.nullSafeEval(variantExpressions.scala:282)
at org.apache.spark.sql.catalyst.expressions.BinaryExpression.eval(Expression.scala:692)
at org.apache.spark.sql.catalyst.expressions.Alias.eval(namedExpressions.scala:159)
at org.apache.spark.sql.catalyst.expressions.InterpretedMutableProjection.apply(InterpretedMutableProjection.scala:89)
at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$$anonfun$apply$48.$anonfun$applyOrElse$83(Optimizer.scala:2231)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79).....

Ich evaluiere die Spark 4 try_variant_get-Methode zur Verarbeitung von Variantentypdaten. Zuerst erstelle ich Beispiele für SQL-Anweisungen.
[code]CREATE TABLE family (
id INT,
data VARIANT
);

INSERT INTO family (id, data)
VALUES
(1, PARSE_JSON('{"name":"Alice","age":30}')),
(2, PARSE_JSON('[1,2,3,4,5]')),
(3, PARSE_JSON('42'));
[/code]
Bei der Ausführung von SQL treten keine Fehler auf. Dann sind die folgenden Codes der Auswahlbefehl mit der Methode try_variant_get
[code]SELECT
id,
try_variant_get(data, '$.name', 'STRING') AS name,
try_variant_get(data, '$.age', 'INT') AS age
FROM
family
WHERE
try_variant_get(data, '$.name', 'STRING') IS NOT NULL;
[/code]
SQL-Ausgabe wurde erfolgreich zurückgegeben. Dann transformiere ich diese SQL-Anweisungen in Java-API-Codes.
[code]SparkSession spark = SparkSession.builder().master("local[*]").appName("VariantExample").getOrCreate();

StructType schema = new StructType()
.add("id", DataTypes.IntegerType)
.add("data", DataTypes.VariantType);

Dataset df = spark.createDataFrame(
Arrays.asList(
RowFactory.create(1, "{\"name\":\"Alice\",\"age\":30}"),
RowFactory.create(2, "[1,2,3,4,5]"),
RowFactory.create(3, "42")
),
schema
);

Dataset df_sel = df.select(
col("id"),
try_variant_get(col("data"), "$.name", "String").alias("name"),
try_variant_get(col("data"), "$.age", "Integer").alias("age")
).where("name IS NOT NULL");

df_sel.printSchema();
df_sel.show();
[/code]
Aber diese Java-Codes lösen die folgenden Ausnahmen aus.
[code]root
|-- id: integer (nullable = true)
|-- name: string (nullable = true)
|-- age: integer (nullable = true)

Exception in thread "main" java.lang.ClassCastException: class java.lang.String cannot be cast to class org.apache.spark.unsafe.types.VariantVal (java.lang.String is in module java.base of loader 'bootstrap'; org.apache.spark.unsafe.types.VariantVal is in unnamed module of loader 'app')
at org.apache.spark.sql.catalyst.expressions.variant.VariantGet.nullSafeEval(variantExpressions.scala:282)
at org.apache.spark.sql.catalyst.expressions.BinaryExpression.eval(Expression.scala:692)
at org.apache.spark.sql.catalyst.expressions.Alias.eval(namedExpressions.scala:159)
at org.apache.spark.sql.catalyst.expressions.InterpretedMutableProjection.apply(InterpretedMutableProjection.scala:89)
at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$$anonfun$apply$48.$anonfun$applyOrElse$83(Optimizer.scala:2231)
at scala.collection.immutable.List.map(List.scala:247)
at scala.collection.immutable.List.map(List.scala:79).....
[/code]
Der Parameter „String“ der Methode try_variant_get weist einige Probleme auf. Aber ich habe keine Ahnung, was mit diesen Java-Codes falsch ist. Bitte teilen Sie mir mit, wie ich diese Fehler beheben kann.