Mobile versionCode: Select all
SparkSession spark = SparkSession.builder().master("local[*]").appName("VariantExample").getOrCreate();
StructType schema = new StructType()
.add("id", DataTypes.IntegerType)
.add("data", DataTypes.VariantType);
Dataset df = spark.createDataFrame(
Arrays.asList(
RowFactory.create(1, "{\"name\":\"Alice\",\"age\":30}"),
RowFactory.create(2, "[1,2,3,4,5]"),
RowFactory.create(3, "42")
),
schema
);
Dataset result = df.select(
col("id"),
try_variant_get(col("data"), "$.name", "string").alias("name"),
try_variant_get(col("data"), "$.age", "int").alias("age")
).where("name IS NOT NULL");
// Show the result
result.show();
spark.stop();
Code: Select all
Exception in thread "main" java.lang.ClassCastException: class java.lang.String cannot be cast to class org.apache.spark.unsafe.types.VariantVal (java.lang.String is in module java.base of loader 'bootstrap'; o r g . a p a c h e . s p a r k . u n s a f e . t y p e s . V a r i a n t V a l i s i n u n n a m e d m o d u l e o f l o a d e r ' a p p ' ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . e x p r e s s i o n s . v a r i a n t . V a r i a n t G e t . n u l l S a f e E v a l ( v a r i a n t E x p r e s s i o n s . s c a l a : 2 8 2 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . e x p r e s s i o n s . B i n a r y E x p r e s s i o n . e v a l ( E x p r e s s i o n . s c a l a : 6 9 2 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . e x p r e s s i o n s . A l i a s . e v a l ( n a m e d E x p r e s s i o n s . s c a l a : 1 5 9 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . e x p r e s s i o n s . I n t e r p r e t e d M u t a b l e P r o j e c t i o n . a p p l y ( I n t e r p r e t e d M u t a b l e P r o j e c t i o n . s c a l a : 8 9 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . o p t i m i z e r . C o n v e r t T o L o c a l R e l a t i o n $ $ a n o n f u n $ a p p l y $ 4 8 . $ a n o n f u n $ a p p l y O r E l s e $ 8 3 ( O p t i m i z e r . s c a l a : 2 2 3 1 ) < b r / > a t s c a l a . c o l l e c t i o n . i m m u t a b l e . L i s t . m a p ( L i s t . s c a l a : 2 4 7 ) < b r / > a t s c a l a . c o l l e c t i o n . i m m u t a b l e . L i s t . m a p ( L i s t . s c a l a : 7 9 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . o p t i m i z e r . C o n v e r t T o L o c a l R e l a t i o n $ $ a n o n f u n $ a p p l y $ 4 8 . a p p l y O r E l s e ( O p t i m i z e r . s c a l a : 2 2 3 1 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . o p t i m i z e r . C o n v e r t T o L o c a l R e l a t i o n $ $ a n o n f u n $ a p p l y $ 4 8 . a p p l y O r E l s e ( O p t i m i z e r . s c a l a : 2 2 2 6 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . $ a n o n f u n $ t r a n s f o r m D o w n W i t h P r u n i n g $ 1 ( T r e e N o d e . s c a l a : 4 7 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . C u r r e n t O r i g i n $ . w i t h O r i g i n ( o r i g i n . s c a l a : 8 6 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . t r a n s f o r m D o w n W i t h P r u n i n g ( T r e e N o d e . s c a l a : 4 7 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . o r g $ a p a c h e $ s p a r k $ s q l $ c a t a l y s t $ p l a n s $ l o g i c a l $ A n a l y s i s H e l p e r $ $ s u p e r $ t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g ( A n a l y s i s H e l p e r . s c a l a : 3 3 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g $ ( A n a l y s i s H e l p e r . s c a l a : 3 2 6 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . $ a n o n f u n $ t r a n s f o r m D o w n W i t h P r u n i n g $ 3 ( T r e e N o d e . s c a l a : 4 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . U n a r y L i k e . m a p C h i l d r e n ( T r e e N o d e . s c a l a : 1 2 2 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . U n a r y L i k e . m a p C h i l d r e n $ ( T r e e N o d e . s c a l a : 1 2 2 4 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . F i l t e r . m a p C h i l d r e n ( b a s i c L o g i c a l O p e r a t o r s . s c a l a : 3 1 8 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . t r a n s f o r m D o w n W i t h P r u n i n g ( T r e e N o d e . s c a l a : 4 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . o r g $ a p a c h e $ s p a r k $ s q l $ c a t a l y s t $ p l a n s $ l o g i c a l $ A n a l y s i s H e l p e r $ $ s u p e r $ t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g ( A n a l y s i s H e l p e r . s c a l a : 3 3 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g $ ( A n a l y s i s H e l p e r . s c a l a : 3 2 6 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . $ a n o n f u n $ t r a n s f o r m D o w n W i t h P r u n i n g $ 3 ( T r e e N o d e . s c a l a : 4 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . U n a r y L i k e . m a p C h i l d r e n ( T r e e N o d e . s c a l a : 1 2 2 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . U n a r y L i k e . m a p C h i l d r e n $ ( T r e e N o d e . s c a l a : 1 2 2 4 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . P r o j e c t . m a p C h i l d r e n ( b a s i c L o g i c a l O p e r a t o r s . s c a l a : 7 2 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . t r a n s f o r m D o w n W i t h P r u n i n g ( T r e e N o d e . s c a l a : 4 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . o r g $ a p a c h e $ s p a r k $ s q l $ c a t a l y s t $ p l a n s $ l o g i c a l $ A n a l y s i s H e l p e r $ $ s u p e r $ t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g ( A n a l y s i s H e l p e r . s c a l a : 3 3 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g $ ( A n a l y s i s H e l p e r . s c a l a : 3 2 6 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . $ a n o n f u n $ t r a n s f o r m D o w n W i t h P r u n i n g $ 3 ( T r e e N o d e . s c a l a : 4 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . U n a r y L i k e . m a p C h i l d r e n ( T r e e N o d e . s c a l a : 1 2 2 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . U n a r y L i k e . m a p C h i l d r e n $ ( T r e e N o d e . s c a l a : 1 2 2 4 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o c a l L i m i t . m a p C h i l d r e n ( b a s i c L o g i c a l O p e r a t o r s . s c a l a : 1 6 6 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . t r a n s f o r m D o w n W i t h P r u n i n g ( T r e e N o d e . s c a l a : 4 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . o r g $ a p a c h e $ s p a r k $ s q l $ c a t a l y s t $ p l a n s $ l o g i c a l $ A n a l y s i s H e l p e r $ $ s u p e r $ t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g ( A n a l y s i s H e l p e r . s c a l a : 3 3 0 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . A n a l y s i s H e l p e r . t r a n s f o r m D o w n W i t h P r u n i n g $ ( A n a l y s i s H e l p e r . s c a l a : 3 2 6 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . L o g i c a l P l a n . t r a n s f o r m D o w n W i t h P r u n i n g ( L o g i c a l P l a n . s c a l a : 3 7 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . T r e e N o d e . $ a n o n f u n $ t r a n s f o r m D o w n W i t h P r u n i n g $ 3 ( T r e e N o d e . s c a l a : 4 7 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . U n a r y L i k e . m a p C h i l d r e n ( T r e e N o d e . s c a l a : 1 2 2 5 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . t r e e s . U n a r y L i k e . m a p C h i l d r e n $ ( T r e e N o d e . s c a l a : 1 2 2 4 ) < b r / > a t o r g . a p a c h e . s p a r k . s q l . c a t a l y s t . p l a n s . l o g i c a l . G l o b a l L i m i t . m a p C h i l d r e n ( b a s i c L o g icalOperators.scala:1646)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:475)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:330)
at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:326)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformWithPruning(TreeNode.scala:436)
at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$.apply(Optimizer.scala:2226)
at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$.apply(Optimizer.scala:2224)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:226)
at scala.collection.immutable.ArraySeq.foldLeft(ArraySeq.scala:222)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:223)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1$adapted(RuleExecutor.scala:215)
at scala.collection.immutable.List.foreach(List.scala:334)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:215)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:186)
at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:89)
at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:186)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$optimizedPlan$1(QueryExecution.scala:167)
at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:138)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:234)
at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:608)
at org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:234)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:742)
at org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:233)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:163)
at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:159)
at org.apache.spark.sql.execution.QueryExecution.assertOptimized(QueryExecution.scala:177)
at org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:197)
at org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:194)
at org.apache.spark.sql.execution.QueryExecution.simpleString(QueryExecution.scala:253)
at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:300)
at org.apache.spark.sql.execution.QueryExecution.explainString(QueryExecution.scala:268)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:142)
at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:267)
at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:118)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:742)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:74)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:222)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:2199)
at org.apache.spark.sql.Dataset.head(Dataset.scala:1350)
at org.apache.spark.sql.api.Dataset.take(Dataset.scala:2640)
at org.apache.spark.sql.Dataset.getRows(Dataset.scala:299)
at org.apache.spark.sql.Dataset.showString(Dataset.scala:335)
at org.apache.spark.sql.Dataset.show(Dataset.scala:569)
at org.apache.spark.sql.api.Dataset.show(Dataset.scala:423)
at org.apache.spark.sql.api.Dataset.show(Dataset.scala:432)
at com.aaa.spark.VariantDataType.main(VariantDataType.java:58)
