{ "_id": "6a1f0c77b401979e7341cb3f", "Type": "Package", "Package": "sparklyr", "Title": "R Interface to Apache Spark", "Version": "1.9.4", "Authors@R": "c(person(given = \"Javier\",\nfamily = \"Luraschi\",\nrole = \"aut\",\nemail = \"jluraschi@gmail.com\"),\nperson(given = \"Kevin\",\nfamily = \"Kuo\",\nrole = \"aut\",\nemail = \"kevin.kuo@rstudio.com\",\ncomment = c(ORCID = \"0000-0001-7803-7901\")),\nperson(given = \"Kevin\",\nfamily = \"Ushey\",\nrole = \"aut\",\nemail = \"kevin@rstudio.com\"),\nperson(given = \"JJ\",\nfamily = \"Allaire\",\nrole = \"aut\",\nemail = \"jj@rstudio.com\"),\nperson(given = \"Samuel\",\nfamily = \"Macedo\",\nrole = \"ctb\",\nemail = \"samuelmacedo@recife.ifpe.edu.br\"),\nperson(given = \"Hossein\",\nfamily = \"Falaki\",\nrole = \"aut\",\nemail = \"hossein@databricks.com\"),\nperson(given = \"Lu\",\nfamily = \"Wang\",\nrole = \"aut\",\nemail = \"lu.wang@databricks.com\"),\nperson(given = \"Andy\",\nfamily = \"Zhang\",\nrole = \"aut\",\nemail = \"yue.zhang@databricks.com\"),\nperson(given = \"Yitao\",\nfamily = \"Li\",\nrole = \"aut\",\nemail = \"yitaoli1990@gmail.com\",\ncomment = c(ORCID = \"0000-0002-1261-905X\")),\nperson(given = \"Jozef\",\nfamily = \"Hajnala\",\nrole = \"ctb\",\nemail = \"jozef.hajnala@gmail.com\"),\nperson(given = \"Maciej\",\nfamily = \"Szymkiewicz\",\nrole = \"ctb\",\nemail = \"mszymkiewicz@gmail.com\",\ncomment = c(ORCID = \"0000-0003-1469-9396\")),\nperson(given = \"Wil\",\nfamily = \"Davis\",\nrole = \"ctb\",\nemail = \"william.davis@worthingtonindustries.com\"),\nperson(given = \"Edgar\",\nfamily = \"Ruiz\",\nrole = c(\"aut\", \"cre\"),\nemail = \"edgar@rstudio.com\"),\nperson(family = \"RStudio\",\nrole = \"cph\"),\nperson(family = \"The Apache Software Foundation\",\nrole = c(\"aut\", \"cph\")))", "Maintainer": "Edgar Ruiz ", "Description": "R interface to Apache Spark, a fast and general engine for\nbig data processing, see . This\npackage supports connecting to local and remote Apache Spark\nclusters, provides a 'dplyr' compatible back-end, and provides\nan interface to Spark's built-in machine learning algorithms.", "License": "Apache License 2.0 | file LICENSE", "URL": "https://spark.posit.co/", "BugReports": "https://github.com/sparklyr/sparklyr/issues", "Encoding": "UTF-8", "RoxygenNote": "7.3.3", "SystemRequirements": "Spark: 2.x, or 3.x, or 4.x", "Collate": "'spark_data_build_types.R' 'arrow_data.R' 'spark_invoke.R'\n'browse_url.R' 'spark_connection.R' 'avro_utils.R'\n'config_settings.R' 'config_spark.R' 'connection_instances.R'\n'connection_progress.R' 'connection_shinyapp.R'\n'spark_version.R' 'connection_spark.R' 'core_arrow.R'\n'core_config.R' 'core_connection.R' 'core_deserialize.R'\n'core_gateway.R' 'core_invoke.R' 'core_jobj.R'\n'core_serialize.R' 'core_utils.R' 'core_worker_config.R'\n'utils.R' 'sql_utils.R' 'data_copy.R' 'data_csv.R'\n'spark_schema_from_rdd.R' 'spark_apply_bundle.R'\n'spark_apply.R' 'tables_spark.R' 'tbl_spark.R' 'spark_sql.R'\n'spark_dataframe.R' 'dplyr_spark.R' 'sdf_interface.R'\n'data_interface.R' 'databricks_connection.R'\n'dbi_spark_connection.R' 'dbi_spark_result.R'\n'dbi_spark_table.R' 'do_spark.R' 'dplyr_do.R' 'dplyr_hof.R'\n'dplyr_join.R' 'dplyr_spark_data.R' 'dplyr_spark_table.R'\n'stratified_sample.R' 'sdf_sql.R' 'dplyr_sql.R'\n'dplyr_sql_translation.R' 'dplyr_verbs.R' 'imports.R'\n'install_spark.R' 'install_spark_versions.R'\n'install_spark_windows.R' 'install_tools.R' 'java.R'\n'jobs_api.R' 'kubernetes_config.R' 'shell_connection.R'\n'livy_connection.R' 'livy_install.R' 'livy_invoke.R'\n'livy_service.R' 'ml_clustering.R'\n'ml_classification_decision_tree_classifier.R'\n'ml_classification_gbt_classifier.R'\n'ml_classification_linear_svc.R'\n'ml_classification_logistic_regression.R'\n'ml_classification_multilayer_perceptron_classifier.R'\n'ml_classification_naive_bayes.R'\n'ml_classification_one_vs_rest.R'\n'ml_classification_random_forest_classifier.R'\n'ml_model_helpers.R' 'ml_clustering_bisecting_kmeans.R'\n'ml_clustering_gaussian_mixture.R' 'ml_clustering_kmeans.R'\n'ml_clustering_lda.R' 'ml_clustering_power_iteration.R'\n'ml_constructor_utils.R' 'ml_evaluate.R'\n'ml_evaluation_clustering.R' 'ml_evaluation_prediction.R'\n'ml_evaluator.R' 'ml_feature_binarizer.R'\n'ml_feature_bucketed_random_projection_lsh.R'\n'ml_feature_bucketizer.R' 'ml_feature_chisq_selector.R'\n'ml_feature_count_vectorizer.R' 'ml_feature_dct.R'\n'ml_feature_sql_transformer.R' 'ml_feature_dplyr_transformer.R'\n'ml_feature_elementwise_product.R'\n'ml_feature_feature_hasher.R' 'ml_feature_hashing_tf.R'\n'ml_feature_idf.R' 'ml_feature_imputer.R'\n'ml_feature_index_to_string.R' 'ml_feature_interaction.R'\n'ml_feature_lsh_utils.R' 'ml_feature_max_abs_scaler.R'\n'ml_feature_min_max_scaler.R' 'ml_feature_minhash_lsh.R'\n'ml_feature_ngram.R' 'ml_feature_normalizer.R'\n'ml_feature_one_hot_encoder.R'\n'ml_feature_one_hot_encoder_estimator.R' 'ml_feature_pca.R'\n'ml_feature_polynomial_expansion.R'\n'ml_feature_quantile_discretizer.R' 'ml_feature_r_formula.R'\n'ml_feature_regex_tokenizer.R' 'ml_feature_robust_scaler.R'\n'ml_feature_standard_scaler.R'\n'ml_feature_stop_words_remover.R' 'ml_feature_string_indexer.R'\n'ml_feature_string_indexer_model.R' 'ml_feature_tokenizer.R'\n'ml_feature_vector_assembler.R' 'ml_feature_vector_indexer.R'\n'ml_feature_vector_slicer.R' 'ml_feature_word2vec.R'\n'ml_fpm_fpgrowth.R' 'ml_fpm_prefixspan.R' 'ml_helpers.R'\n'ml_mapping_tables.R' 'ml_metrics.R' 'ml_model_als.R'\n'ml_model_bisecting_kmeans.R' 'ml_model_constructors.R'\n'ml_model_decision_tree.R' 'ml_model_gaussian_mixture.R'\n'ml_model_generalized_linear_regression.R'\n'ml_model_gradient_boosted_trees.R'\n'ml_model_isotonic_regression.R' 'ml_model_kmeans.R'\n'ml_model_lda.R' 'ml_model_linear_regression.R'\n'ml_model_linear_svc.R' 'ml_model_logistic_regression.R'\n'ml_model_naive_bayes.R' 'ml_model_one_vs_rest.R'\n'ml_model_random_forest.R' 'ml_model_utils.R'\n'ml_param_utils.R' 'ml_persistence.R' 'ml_pipeline.R'\n'ml_pipeline_utils.R' 'ml_print_utils.R'\n'ml_recommendation_als.R'\n'ml_regression_aft_survival_regression.R'\n'ml_regression_decision_tree_regressor.R'\n'ml_regression_gbt_regressor.R'\n'ml_regression_generalized_linear_regression.R'\n'ml_regression_isotonic_regression.R'\n'ml_regression_linear_regression.R'\n'ml_regression_random_forest_regressor.R' 'ml_stat.R'\n'ml_summary.R' 'ml_transformation_methods.R'\n'ml_transformer_and_estimator.R' 'ml_tuning.R'\n'ml_tuning_cross_validator.R'\n'ml_tuning_train_validation_split.R' 'ml_utils.R'\n'ml_validator_utils.R' 'mutation.R' 'na_actions.R'\n'new_model_multilayer_perceptron.R' 'params_validator.R'\n'precondition.R' 'project_template.R' 'qubole_connection.R'\n'reexports.R' 'sdf_dim.R' 'sdf_distinct.R' 'sdf_ml.R'\n'sdf_saveload.R' 'sdf_sequence.R' 'sdf_stat.R'\n'sdf_streaming.R' 'tidyr_utils.R' 'sdf_unnest_longer.R'\n'sdf_wrapper.R' 'sdf_unnest_wider.R' 'sdf_utils.R'\n'spark_compile.R' 'spark_context_config.R' 'spark_extensions.R'\n'spark_gateway.R' 'spark_gen_embedded_sources.R'\n'spark_globals.R' 'spark_hive.R' 'spark_home.R' 'spark_ide.R'\n'spark_submit.R' 'spark_update_embedded_sources.R'\n'spark_utils.R' 'spark_verify_embedded_sources.R'\n'stream_data.R' 'stream_job.R' 'stream_operations.R'\n'stream_shiny.R' 'stream_view.R' 'synapse_connection.R'\n'test_connection.R' 'tidiers_ml_aft_survival_regression.R'\n'tidiers_ml_als.R' 'tidiers_ml_isotonic_regression.R'\n'tidiers_ml_lda.R' 'tidiers_ml_linear_models.R'\n'tidiers_ml_logistic_regression.R'\n'tidiers_ml_multilayer_perceptron.R' 'tidiers_ml_naive_bayes.R'\n'tidiers_ml_svc_models.R' 'tidiers_ml_tree_models.R'\n'tidiers_ml_unsupervised_models.R' 'tidiers_pca.R'\n'tidiers_utils.R' 'tidyr_fill.R' 'tidyr_nest.R'\n'tidyr_pivot_utils.R' 'tidyr_pivot_longer.R'\n'tidyr_pivot_wider.R' 'tidyr_separate.R' 'tidyr_unite.R'\n'tidyr_unnest.R' 'tune-grid-spark.R' 'worker_apply.R'\n'worker_connect.R' 'worker_connection.R' 'worker_invoke.R'\n'worker_log.R' 'worker_main.R' 'yarn_cluster.R' 'yarn_config.R'\n'yarn_ui.R' 'zzz.R'", "Config/pak/sysreqs": "libicu-dev libxml2-dev libssl-dev", "Repository": "https://sparklyr.r-universe.dev", "Date/Publication": "2026-04-17 21:07:44 UTC", "RemoteUrl": "https://github.com/sparklyr/sparklyr", "RemoteRef": "HEAD", "RemoteSha": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "NeedsCompilation": "no", "Packaged": { "Date": "2026-05-17 09:31:57 UTC", "User": "root" }, "Author": "Javier Luraschi [aut],\nKevin Kuo [aut] (ORCID: ),\nKevin Ushey [aut],\nJJ Allaire [aut],\nSamuel Macedo [ctb],\nHossein Falaki [aut],\nLu Wang [aut],\nAndy Zhang [aut],\nYitao Li [aut] (ORCID: ),\nJozef Hajnala [ctb],\nMaciej Szymkiewicz [ctb] (ORCID:\n),\nWil Davis [ctb],\nEdgar Ruiz [aut, cre],\nRStudio [cph],\nThe Apache Software Foundation [aut, cph]", "MD5sum": "b50afb4fc25411dd29cf735520759bb2", "_user": "sparklyr", "_type": "src", "_file": "sparklyr_1.9.4.tar.gz", "_fileid": "fe8cef0db94523a4414a3d6f509873225dec31fc8322651301a6f5e008bdcec7", "_filesize": 4164017, "_sha256": "fe8cef0db94523a4414a3d6f509873225dec31fc8322651301a6f5e008bdcec7", "_created": "2026-05-17T09:31:57.000Z", "_published": "2026-06-02T17:01:43.491Z", "_distro": "noble", "_jobs": [ { "job": 79126801548, "time": 192, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "7041270259" }, { "job": 79126802122, "time": 190, "config": "linux-release-x86_64", "r": "4.6.0", "check": "OK", "artifact": "7041269986" }, { "job": 79126802131, "time": 184, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "7041265949" }, { "job": 79126801565, "time": 215, "config": "macos-release-arm64", "r": "4.6.0", "check": "OK", "artifact": "7041269196" }, { "job": 79126801383, "time": 255, "config": "source", "r": "4.6.0", "check": "OK", "artifact": "7041247336" }, { "job": 79126801038, "time": 135, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "7364498909" }, { "job": 79126801607, "time": 157, "config": "windows-devel", "r": "4.7.0", "check": "OK", "artifact": "7041266251" }, { "job": 79126801702, "time": 153, "config": "windows-oldrel", "r": "4.5.3", "check": "OK", "artifact": "7041265652" }, { "job": 79126802135, "time": 162, "config": "windows-release", "r": "4.6.0", "check": "OK", "artifact": "7041266646" } ], "_buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290", "_status": "success", "_host": "GitHub-Actions", "_upstream": "https://github.com/sparklyr/sparklyr", "_commit": { "id": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "author": "Edgar Ruiz <77294576+edgararuiz@users.noreply.github.com>", "committer": "GitHub ", "message": "Merge pull request #3514 from sparklyr/updates\n\nRelease prep", "time": 1776460064 }, "_maintainer": { "name": "Edgar Ruiz", "email": "edgar@rstudio.com", "login": "edgararuiz-zz", "description": "", "uuid": 7875923 }, "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 3.2", "role": "Depends" }, { "package": "config", "version": ">= 0.2", "role": "Imports" }, { "package": "DBI", "version": ">= 1.0.0", "role": "Imports" }, { "package": "dbplyr", "version": ">= 2.5.0", "role": "Imports" }, { "package": "dplyr", "version": ">= 1.0.9", "role": "Imports" }, { "package": "generics", "role": "Imports" }, { "package": "globals", "role": "Imports" }, { "package": "glue", "role": "Imports" }, { "package": "httr", "version": ">= 1.2.1", "role": "Imports" }, { "package": "jsonlite", "version": ">= 1.4", "role": "Imports" }, { "package": "methods", "role": "Imports" }, { "package": "openssl", "version": ">= 0.8", "role": "Imports" }, { "package": "purrr", "role": "Imports" }, { "package": "rlang", "version": ">= 0.1.4", "role": "Imports" }, { "package": "rstudioapi", "version": ">= 0.10", "role": "Imports" }, { "package": "tidyr", "version": ">= 1.2.0", "role": "Imports" }, { "package": "tidyselect", "role": "Imports" }, { "package": "uuid", "role": "Imports" }, { "package": "vctrs", "role": "Imports" }, { "package": "withr", "role": "Imports" }, { "package": "xml2", "role": "Imports" }, { "package": "arrow", "version": ">= 0.17.0", "role": "Suggests" }, { "package": "broom", "role": "Suggests" }, { "package": "diffobj", "role": "Suggests" }, { "package": "foreach", "role": "Suggests" }, { "package": "ggplot2", "role": "Suggests" }, { "package": "iterators", "role": "Suggests" }, { "package": "janeaustenr", "role": "Suggests" }, { "package": "Lahman", "role": "Suggests" }, { "package": "mlbench", "role": "Suggests" }, { "package": "nnet", "role": "Suggests" }, { "package": "nycflights13", "role": "Suggests" }, { "package": "R6", "role": "Suggests" }, { "package": "r2d3", "role": "Suggests" }, { "package": "RCurl", "role": "Suggests" }, { "package": "reshape2", "role": "Suggests" }, { "package": "shiny", "version": ">= 1.0.1", "role": "Suggests" }, { "package": "parsnip", "role": "Suggests" }, { "package": "testthat", "role": "Suggests" }, { "package": "rprojroot", "role": "Suggests" } ], "_owner": "sparklyr", "_selfowned": true, "_usedby": 24, "_updates": [ { "week": "2025-25", "n": 2 }, { "week": "2025-26", "n": 4 }, { "week": "2025-27", "n": 2 }, { "week": "2025-31", "n": 1 }, { "week": "2025-33", "n": 1 }, { "week": "2025-40", "n": 2 }, { "week": "2025-45", "n": 1 }, { "week": "2025-47", "n": 1 }, { "week": "2026-05", "n": 1 }, { "week": "2026-16", "n": 2 } ], "_tags": [ { "name": "v1.9.1", "date": "2025-06-30" }, { "name": "v1.9.2", "date": "2025-10-03" }, { "name": "v1.9.4", "date": "2026-04-17" } ], "_topics": [ "apache-spark", "distributed", "dplyr", "ide", "livy", "machine-learning", "remote-clusters", "spark", "sparklyr" ], "_stars": 969, "_contributors": [ { "user": "javierluraschi", "count": 3887, "uuid": 3478847 }, { "user": "kevinykuo", "count": 1540, "uuid": 5582151 }, { "user": "edgararuiz", "count": 920, "uuid": 77294576 }, { "user": "kevinushey", "count": 519, "uuid": 1976582 }, { "user": "jjallaire", "count": 330, "uuid": 104391 }, { "user": "lu-wang-dl", "count": 27, "uuid": 38018689 }, { "user": "falaki", "count": 22, "uuid": 512364 }, { "user": "mattpollock", "count": 18, "uuid": 9449081 }, { "user": "alibell", "count": 15, "uuid": 13646493 }, { "user": "jozefhajnala", "count": 14, "uuid": 23148397 }, { "user": "chezou", "count": 12, "uuid": 916653 }, { "user": "nealrichardson", "count": 11, "uuid": 2975928 }, { "user": "t-kalinowski", "count": 11, "uuid": 8462255 }, { "user": "loquats", "count": 10, "uuid": 6992972 }, { "user": "jmcphers", "count": 8, "uuid": 470418 }, { "user": "richierocks", "count": 8, "uuid": 197589 }, { "user": "jimhester", "count": 7, "uuid": 205275 }, { "user": "shabbybanks", "count": 6, "uuid": 23323752 }, { "user": "mtoto", "count": 6, "uuid": 10297043 }, { "user": "marcinkosinski", "count": 6, "uuid": 6773454 }, { "user": "yutannihilation", "count": 6, "uuid": 1978793 }, { "user": "lgongmsft", "count": 5, "uuid": 44271368 }, { "user": "zacdav-db", "count": 5, "uuid": 80654433 }, { "user": "agrawalamey", "count": 4, "uuid": 15227113 }, { "user": "bo-zhou-ms", "count": 4, "uuid": 156932676 }, { "user": "rgbkrk", "count": 4, "uuid": 836375 }, { "user": "gregleleu", "count": 4, "uuid": 33321678 }, { "user": "zero323", "count": 4, "uuid": 1554276 }, { "user": "trestletech", "count": 4, "uuid": 1593639 }, { "user": "jspiewak", "count": 4, "uuid": 399516 }, { "user": "edgararuiz-zz", "count": 3, "uuid": 7875923 }, { "user": "davisvaughan", "count": 3, "uuid": 19150088 }, { "user": "mrbago", "count": 3, "uuid": 223219 }, { "user": "amruthashok", "count": 3, "uuid": 192104283 }, { "user": "martinstuder", "count": 2, "uuid": 1091843 }, { "user": "russellpierce", "count": 2, "uuid": 1847158 }, { "user": "mnatter", "count": 2, "uuid": 5270131 }, { "user": "cderv", "count": 2, "uuid": 6791940 }, { "user": "faviovazquez", "count": 2, "uuid": 10162068 }, { "user": "jameslamb", "count": 2, "uuid": 7608904 }, { "user": "smingerson", "count": 2, "uuid": 31328867 }, { "user": "awblocker", "count": 1, "uuid": 44618 }, { "user": "teramonagi", "count": 1, "uuid": 683736 }, { "user": "pgramme", "count": 1, "uuid": 4621582 }, { "user": "rishabhbhardwaj", "count": 1, "uuid": 4653910 }, { "user": "sshuklao", "count": 1, "uuid": 17142187 }, { "user": "sipemu", "count": 1, "uuid": 1296999 }, { "user": "tdhock", "count": 1, "uuid": 932850 }, { "user": "tristanz", "count": 1, "uuid": 290978 }, { "user": "wkdavis", "count": 1, "uuid": 10822247 }, { "user": "pursuitofdatascience", "count": 1, "uuid": 54338793 }, { "user": "chenye", "count": 1, "uuid": 5711237 }, { "user": "xuzikun2003", "count": 1, "uuid": 25806935 }, { "user": "gbouzill", "count": 1, "uuid": 33899340 }, { "user": "jangorecki", "count": 1, "uuid": 3627377 }, { "user": "mzorko", "count": 1, "uuid": 51319014 }, { "user": "nathaneastwood", "count": 1, "uuid": 9799530 }, { "user": "weichisyu", "count": 1, "uuid": 119550682 }, { "user": "ybjoony", "count": 1, "uuid": 1686996 }, { "user": "amruth-ashok", "count": 1, "uuid": 73311225 }, { "user": "andrew-christianson", "count": 1, "uuid": 3965903 }, { "user": "bbrewington", "count": 1, "uuid": 10573749 }, { "user": "bpvgoncalves", "count": 1, "uuid": 7214777 }, { "user": "meztez", "count": 1, "uuid": 31039074 }, { "user": "davidkretch", "count": 1, "uuid": 4283778 }, { "user": "dalloliogm", "count": 1, "uuid": 14500 }, { "user": "gfrmin", "count": 1, "uuid": 416374 }, { "user": "gaborcsardi", "count": 1, "uuid": 660288 }, { "user": "ibrahimhaddad", "count": 1, "uuid": 1656002 }, { "user": "jlee2000", "count": 1, "uuid": 41459352 }, { "user": "jmbarbone", "count": 1, "uuid": 38573843 }, { "user": "joveyuan-db", "count": 1, "uuid": 67606568 }, { "user": "katrinleinweber", "count": 1, "uuid": 9948149 }, { "user": "lionel-", "count": 1, "uuid": 4465050 }, { "user": "farrellm", "count": 1, "uuid": 165717 }, { "user": "lawremi", "count": 1, "uuid": 158190 } ], "_userbio": { "uuid": 58250306, "type": "organization", "name": "sparklyr", "description": "R Interface to Apache Spark" }, "_downloads": { "count": 36317, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/sparklyr" }, "_mentions": 4, "_devurl": "https://github.com/sparklyr/sparklyr", "_searchresults": 3872, "_rbuild": "4.6.0", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "extra/sparklyr.html", "manual.pdf" ], "_homeurl": "https://github.com/sparklyr/sparklyr", "_realowner": "sparklyr", "_cranurl": true, "_releases": [ { "version": "0.4", "date": "2016-09-24" }, { "version": "0.5", "date": "2016-12-18" }, { "version": "0.5.1", "date": "2016-12-19" }, { "version": "0.5.2", "date": "2017-02-16" }, { "version": "0.5.3", "date": "2017-03-09" }, { "version": "0.5.4", "date": "2017-04-25" }, { "version": "0.5.5", "date": "2017-05-26" }, { "version": "0.5.6", "date": "2017-06-10" }, { "version": "0.6.0", "date": "2017-07-29" }, { "version": "0.6.1", "date": "2017-08-06" }, { "version": "0.6.2", "date": "2017-08-13" }, { "version": "0.6.3", "date": "2017-09-19" }, { "version": "0.6.4", "date": "2017-11-02" }, { "version": "0.7.0", "date": "2018-01-23" }, { "version": "0.8.0", "date": "2018-05-01" }, { "version": "0.8.1", "date": "2018-05-02" }, { "version": "0.8.2", "date": "2018-05-06" }, { "version": "0.8.3", "date": "2018-05-12" }, { "version": "0.8.4", "date": "2018-05-25" }, { "version": "0.9.1", "date": "2018-09-27" }, { "version": "0.9.2", "date": "2018-10-17" }, { "version": "0.9.3", "date": "2018-11-29" }, { "version": "0.9.4", "date": "2019-01-09" }, { "version": "1.0.0", "date": "2019-02-25" }, { "version": "1.0.1", "date": "2019-05-17" }, { "version": "1.0.2", "date": "2019-07-04" }, { "version": "1.0.3", "date": "2019-09-15" }, { "version": "1.0.4", "date": "2019-10-05" }, { "version": "1.0.5", "date": "2019-11-14" }, { "version": "1.1.0", "date": "2020-01-11" }, { "version": "1.2.0", "date": "2020-04-20" }, { "version": "1.3.0", "date": "2020-06-27" }, { "version": "1.3.1", "date": "2020-07-09" }, { "version": "1.4.0", "date": "2020-09-16" }, { "version": "1.5.0", "date": "2020-11-26" }, { "version": "1.5.1", "date": "2020-12-01" }, { "version": "1.5.2", "date": "2020-12-12" }, { "version": "1.6.0", "date": "2021-03-09" }, { "version": "1.6.1", "date": "2021-03-23" }, { "version": "1.6.2", "date": "2021-03-30" }, { "version": "1.6.3", "date": "2021-06-01" }, { "version": "1.7.0", "date": "2021-06-10" }, { "version": "1.7.1", "date": "2021-06-17" }, { "version": "1.7.2", "date": "2021-09-16" }, { "version": "1.7.3", "date": "2021-11-30" }, { "version": "1.7.4", "date": "2022-01-08" }, { "version": "1.7.5", "date": "2022-02-02" }, { "version": "1.7.6", "date": "2022-05-26" }, { "version": "1.7.7", "date": "2022-06-07" }, { "version": "1.7.8", "date": "2022-08-16" }, { "version": "1.7.9", "date": "2022-12-08" }, { "version": "1.8.0", "date": "2023-03-07" }, { "version": "1.8.1", "date": "2023-03-22" }, { "version": "1.8.2", "date": "2023-07-01" }, { "version": "1.8.3", "date": "2023-09-02" }, { "version": "1.8.4", "date": "2023-10-30" }, { "version": "1.8.5", "date": "2024-03-25" }, { "version": "1.8.6", "date": "2024-04-29" }, { "version": "1.9.0", "date": "2025-03-22" }, { "version": "1.9.1", "date": "2025-06-30" }, { "version": "1.9.2", "date": "2025-10-05" }, { "version": "1.9.3", "date": "2025-11-19" }, { "version": "1.9.4", "date": "2026-04-18" } ], "_exports": [ "%->%", "%>%", "arrow_enabled_object", "augment", "collect", "collect_from_rds", "compile_package_jars", "connection_config", "connection_is_open", "connection_spark_shinyapp", "copy_to", "distinct", "download_scalac", "fill", "filter", "find_scalac", "ft_binarizer", "ft_bucketed_random_projection_lsh", "ft_bucketizer", "ft_chisq_selector", "ft_count_vectorizer", "ft_dct", "ft_discrete_cosine_transform", "ft_dplyr_transformer", "ft_elementwise_product", "ft_feature_hasher", "ft_hashing_tf", "ft_idf", "ft_imputer", "ft_index_to_string", "ft_interaction", "ft_max_abs_scaler", "ft_min_max_scaler", "ft_minhash_lsh", "ft_ngram", "ft_normalizer", "ft_one_hot_encoder", "ft_one_hot_encoder_estimator", "ft_pca", "ft_polynomial_expansion", "ft_quantile_discretizer", "ft_r_formula", "ft_regex_tokenizer", "ft_robust_scaler", "ft_sql_transformer", "ft_standard_scaler", "ft_stop_words_remover", "ft_string_indexer", "ft_string_indexer_model", "ft_tokenizer", "ft_vector_assembler", "ft_vector_indexer", "ft_vector_slicer", "ft_word2vec", "full_join", "get_spark_sql_catalog_implementation", "glance", "hive_context", "hive_context_config", "hof_aggregate", "hof_array_sort", "hof_exists", "hof_filter", "hof_forall", "hof_map_filter", "hof_map_zip_with", "hof_transform", "hof_transform_keys", "hof_transform_values", "hof_zip_with", "inner_join", "invoke", "invoke_method", "invoke_new", "invoke_static", "is_ml_estimator", "is_ml_transformer", "j_invoke", "j_invoke_method", "j_invoke_new", "j_invoke_static", "jarray", "java_context", "jfloat", "jfloat_array", "jobj_class", "jobj_set_param", "left_join", "livy_available_versions", "livy_config", "livy_home_dir", "livy_install", "livy_install_dir", "livy_installed_versions", "livy_service_start", "livy_service_stop", "ml_add_stage", "ml_aft_survival_regression", "ml_als", "ml_approx_nearest_neighbors", "ml_approx_similarity_join", "ml_association_rules", "ml_binary_classification_eval", "ml_binary_classification_evaluator", "ml_bisecting_kmeans", "ml_call_constructor", "ml_chisquare_test", "ml_classification_eval", "ml_clustering_evaluator", "ml_clustering_pipeline", "ml_compute_cost", "ml_compute_silhouette_measure", "ml_construct_model_clustering", "ml_construct_model_supervised", "ml_corr", "ml_cross_validator", "ml_decision_tree", "ml_decision_tree_classifier", "ml_decision_tree_regressor", "ml_default_stop_words", "ml_describe_topics", "ml_evaluate", "ml_feature_importances", "ml_find_synonyms", "ml_fit", "ml_fit_and_transform", "ml_fpgrowth", "ml_freq_itemsets", "ml_freq_seq_patterns", "ml_gaussian_mixture", "ml_gbt_classifier", "ml_gbt_regressor", "ml_generalized_linear_regression", "ml_gradient_boosted_trees", "ml_is_set", "ml_isotonic_regression", "ml_kmeans", "ml_labels", "ml_lda", "ml_linear_regression", "ml_linear_svc", "ml_load", "ml_log_likelihood", "ml_log_perplexity", "ml_logistic_regression", "ml_metrics_binary", "ml_metrics_multiclass", "ml_metrics_regression", "ml_model_data", "ml_multiclass_classification_evaluator", "ml_multilayer_perceptron", "ml_multilayer_perceptron_classifier", "ml_naive_bayes", "ml_one_vs_rest", "ml_param", "ml_param_map", "ml_params", "ml_pca", "ml_pipeline", "ml_power_iteration", "ml_predict", "ml_prefixspan", "ml_random_forest", "ml_random_forest_classifier", "ml_random_forest_regressor", "ml_recommend", "ml_regression_evaluator", "ml_save", "ml_stage", "ml_stages", "ml_standardize_formula", "ml_sub_models", "ml_summary", "ml_supervised_pipeline", "ml_survival_regression", "ml_topics_matrix", "ml_train_validation_split", "ml_transform", "ml_tree_feature_importance", "ml_uid", "ml_validation_metrics", "ml_vocabulary", "mutate", "na.replace", "nest", "new_ml_classification_model", "new_ml_classifier", "new_ml_clustering_model", "new_ml_estimator", "new_ml_model", "new_ml_model_classification", "new_ml_model_clustering", "new_ml_model_prediction", "new_ml_model_regression", "new_ml_prediction_model", "new_ml_predictor", "new_ml_probabilistic_classification_model", "new_ml_probabilistic_classifier", "new_ml_transformer", "pivot_longer", "pivot_wider", "print_jobj", "quote_sql_name", "random_string", "reactiveSpark", "register_extension", "registerDoSpark", "registered_extensions", "replace_na", "right_join", "sdf_along", "sdf_bind_cols", "sdf_bind_rows", "sdf_broadcast", "sdf_checkpoint", "sdf_coalesce", "sdf_collect", "sdf_copy_to", "sdf_crosstab", "sdf_debug_string", "sdf_describe", "sdf_dim", "sdf_distinct", "sdf_drop_duplicates", "sdf_expand_grid", "sdf_fit", "sdf_fit_and_transform", "sdf_from_avro", "sdf_import", "sdf_is_streaming", "sdf_last_index", "sdf_len", "sdf_load_parquet", "sdf_load_table", "sdf_ncol", "sdf_nrow", "sdf_num_partitions", "sdf_partition", "sdf_partition_sizes", "sdf_persist", "sdf_pivot", "sdf_predict", "sdf_project", "sdf_quantile", "sdf_random_split", "sdf_rbeta", "sdf_rbinom", "sdf_rcauchy", "sdf_rchisq", "sdf_read_column", "sdf_register", "sdf_repartition", "sdf_residuals", "sdf_rexp", "sdf_rgamma", "sdf_rgeom", "sdf_rhyper", "sdf_rlnorm", "sdf_rnorm", "sdf_rpois", "sdf_rt", "sdf_runif", "sdf_rweibull", "sdf_sample", "sdf_save_parquet", "sdf_save_table", "sdf_schema", "sdf_separate_column", "sdf_seq", "sdf_sort", "sdf_sql", "sdf_to_avro", "sdf_transform", "sdf_unnest_longer", "sdf_unnest_wider", "sdf_weighted_sample", "sdf_with_sequential_id", "sdf_with_unique_id", "select", "separate", "slice_.tbl_spark", "spark_adaptive_query_execution", "spark_advisory_shuffle_partition_size", "spark_apply", "spark_apply_bundle", "spark_apply_log", "spark_auto_broadcast_join_threshold", "spark_available_versions", "spark_coalesce_initial_num_partitions", "spark_coalesce_min_num_partitions", "spark_coalesce_shuffle_partitions", "spark_compilation_spec", "spark_compile", "spark_config", "spark_config_exists", "spark_config_kubernetes", "spark_config_packages", "spark_config_settings", "spark_config_value", "spark_connect", "spark_connect_method", "spark_connection", "spark_connection_find", "spark_connection_is_open", "spark_context", "spark_context_config", "spark_dataframe", "spark_default_compilation_spec", "spark_default_version", "spark_dependency", "spark_dependency_fallback", "spark_disconnect", "spark_disconnect_all", "spark_extension", "spark_get_checkpoint_dir", "spark_get_java", "spark_home_dir", "spark_home_set", "spark_ide_columns", "spark_ide_connection_actions", "spark_ide_connection_closed", "spark_ide_connection_open", "spark_ide_connection_updated", "spark_ide_objects", "spark_ide_preview", "spark_insert_table", "spark_install", "spark_install_dir", "spark_install_find", "spark_install_tar", "spark_installed_versions", "spark_integ_test_skip", "spark_jobj", "spark_last_error", "spark_load_table", "spark_log", "spark_pipeline_stage", "spark_read", "spark_read_avro", "spark_read_binary", "spark_read_csv", "spark_read_delta", "spark_read_image", "spark_read_jdbc", "spark_read_json", "spark_read_libsvm", "spark_read_orc", "spark_read_parquet", "spark_read_source", "spark_read_table", "spark_read_text", "spark_save_table", "spark_session", "spark_session_config", "spark_set_checkpoint_dir", "spark_submit", "spark_table_name", "spark_uninstall", "spark_version", "spark_version_from_home", "spark_versions", "spark_web", "spark_write", "spark_write_avro", "spark_write_csv", "spark_write_delta", "spark_write_jdbc", "spark_write_json", "spark_write_orc", "spark_write_parquet", "spark_write_rds", "spark_write_source", "spark_write_table", "spark_write_text", "sparklyr_get_backend_port", "src_databases", "stream_find", "stream_generate_test", "stream_id", "stream_lag", "stream_name", "stream_read_cloudfiles", "stream_read_csv", "stream_read_delta", "stream_read_json", "stream_read_kafka", "stream_read_orc", "stream_read_parquet", "stream_read_socket", "stream_read_table", "stream_read_text", "stream_render", "stream_stats", "stream_stop", "stream_trigger_continuous", "stream_trigger_interval", "stream_view", "stream_watermark", "stream_write_console", "stream_write_csv", "stream_write_delta", "stream_write_json", "stream_write_kafka", "stream_write_memory", "stream_write_orc", "stream_write_parquet", "stream_write_table", "stream_write_text", "tbl_cache", "tbl_change_db", "tbl_uncache", "tidy", "tune_grid_spark", "unite", "unnest", "worker_spark_apply_unbundle" ], "_help": [ { "page": "sub-.tbl_spark", "title": "Subsetting operator for Spark dataframe", "topics": [ "[.tbl_spark" ] }, { "page": "grapes-greater-than-grapes", "title": "Infix operator for composing a lambda expression", "topics": [ "%->%" ] }, { "page": "checkpoint_directory", "title": "Set/Get Spark checkpoint directory", "topics": [ "checkpoint_directory", "spark_get_checkpoint_dir", "spark_set_checkpoint_dir" ] }, { "page": "collect_from_rds", "title": "Collect Spark data serialized in RDS format into R", "concept": [ "Spark serialization routines" ], "topics": [ "collect_from_rds" ] }, { "page": "compile_package_jars", "title": "Compile Scala sources into a Java Archive (jar)", "topics": [ "compile_package_jars" ] }, { "page": "connection_config", "title": "Read configuration values for a connection", "topics": [ "connection_config" ] }, { "page": "copy_to.spark_connection", "title": "Copy an R Data Frame to Spark", "topics": [ "copy_to.spark_connection" ] }, { "page": "distinct", "title": "Distinct", "topics": [ "distinct" ] }, { "page": "download_scalac", "title": "Downloads default Scala Compilers", "topics": [ "download_scalac" ] }, { "page": "dplyr_hof", "title": "dplyr wrappers for Apache Spark higher order functions", "topics": [ "dplyr_hof" ] }, { "page": "ensure", "title": "Enforce Specific Structure for R Objects", "topics": [ "ensure" ] }, { "page": "fill", "title": "Fill", "topics": [ "fill" ] }, { "page": "filter", "title": "Filter", "topics": [ "filter" ] }, { "page": "find_scalac", "title": "Discover the Scala Compiler", "topics": [ "find_scalac" ] }, { "page": "ft_binarizer", "title": "Feature Transformation - Binarizer (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_binarizer" ] }, { "page": "ft_bucketizer", "title": "Feature Transformation - Bucketizer (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_bucketizer" ] }, { "page": "ft_chisq_selector", "title": "Feature Transformation - ChiSqSelector (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_chisq_selector" ] }, { "page": "ft_count_vectorizer", "title": "Feature Transformation - CountVectorizer (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_count_vectorizer", "ml_vocabulary" ] }, { "page": "ft_dct", "title": "Feature Transformation - Discrete Cosine Transform (DCT) (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_dct", "ft_discrete_cosine_transform" ] }, { "page": "ft_elementwise_product", "title": "Feature Transformation - ElementwiseProduct (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_elementwise_product" ] }, { "page": "ft_feature_hasher", "title": "Feature Transformation - FeatureHasher (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_feature_hasher" ] }, { "page": "ft_hashing_tf", "title": "Feature Transformation - HashingTF (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_hashing_tf" ] }, { "page": "ft_idf", "title": "Feature Transformation - IDF (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_idf" ] }, { "page": "ft_imputer", "title": "Feature Transformation - Imputer (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_imputer" ] }, { "page": "ft_index_to_string", "title": "Feature Transformation - IndexToString (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_index_to_string" ] }, { "page": "ft_interaction", "title": "Feature Transformation - Interaction (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_interaction" ] }, { "page": "ft_lsh", "title": "Feature Transformation - LSH (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_bucketed_random_projection_lsh", "ft_lsh", "ft_minhash_lsh" ] }, { "page": "ft_lsh_utils", "title": "Utility functions for LSH models", "topics": [ "ft_lsh_utils", "ml_approx_nearest_neighbors", "ml_approx_similarity_join" ] }, { "page": "ft_max_abs_scaler", "title": "Feature Transformation - MaxAbsScaler (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_max_abs_scaler" ] }, { "page": "ft_min_max_scaler", "title": "Feature Transformation - MinMaxScaler (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_min_max_scaler" ] }, { "page": "ft_ngram", "title": "Feature Transformation - NGram (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_ngram" ] }, { "page": "ft_normalizer", "title": "Feature Transformation - Normalizer (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_normalizer" ] }, { "page": "ft_one_hot_encoder", "title": "Feature Transformation - OneHotEncoder (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_one_hot_encoder" ] }, { "page": "ft_one_hot_encoder_estimator", "title": "Feature Transformation - OneHotEncoderEstimator (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_one_hot_encoder_estimator" ] }, { "page": "ft_pca", "title": "Feature Transformation - PCA (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_pca", "ml_pca" ] }, { "page": "ft_polynomial_expansion", "title": "Feature Transformation - PolynomialExpansion (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_polynomial_expansion" ] }, { "page": "ft_quantile_discretizer", "title": "Feature Transformation - QuantileDiscretizer (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_quantile_discretizer" ] }, { "page": "ft_r_formula", "title": "Feature Transformation - RFormula (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_r_formula" ] }, { "page": "ft_regex_tokenizer", "title": "Feature Transformation - RegexTokenizer (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_regex_tokenizer" ] }, { "page": "ft_robust_scaler", "title": "Feature Transformation - RobustScaler (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_robust_scaler" ] }, { "page": "sql-transformer", "title": "Feature Transformation - SQLTransformer", "concept": [ "feature transformers" ], "topics": [ "ft_dplyr_transformer", "ft_sql_transformer" ] }, { "page": "ft_standard_scaler", "title": "Feature Transformation - StandardScaler (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_standard_scaler" ] }, { "page": "ft_stop_words_remover", "title": "Feature Transformation - StopWordsRemover (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_stop_words_remover" ] }, { "page": "ft_string_indexer", "title": "Feature Transformation - StringIndexer (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_string_indexer", "ft_string_indexer_model", "ml_labels" ] }, { "page": "ft_tokenizer", "title": "Feature Transformation - Tokenizer (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_tokenizer" ] }, { "page": "ft_vector_assembler", "title": "Feature Transformation - VectorAssembler (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_vector_assembler" ] }, { "page": "ft_vector_indexer", "title": "Feature Transformation - VectorIndexer (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_vector_indexer" ] }, { "page": "ft_vector_slicer", "title": "Feature Transformation - VectorSlicer (Transformer)", "concept": [ "feature transformers" ], "topics": [ "ft_vector_slicer" ] }, { "page": "ft_word2vec", "title": "Feature Transformation - Word2Vec (Estimator)", "concept": [ "feature transformers" ], "topics": [ "ft_word2vec", "ml_find_synonyms" ] }, { "page": "full_join", "title": "Full join", "topics": [ "full_join" ] }, { "page": "generic_call_interface", "title": "Generic Call Interface", "topics": [ "generic_call_interface" ] }, { "page": "get_spark_sql_catalog_implementation", "title": "Retrieve the Spark connection's SQL catalog implementation property", "topics": [ "get_spark_sql_catalog_implementation" ] }, { "page": "hive_context_config", "title": "Runtime configuration interface for Hive", "topics": [ "hive_context_config" ] }, { "page": "hof_aggregate", "title": "Apply Aggregate Function to Array Column", "topics": [ "hof_aggregate" ] }, { "page": "hof_array_sort", "title": "Sorts array using a custom comparator", "topics": [ "hof_array_sort" ] }, { "page": "hof_exists", "title": "Determine Whether Some Element Exists in an Array Column", "topics": [ "hof_exists" ] }, { "page": "hof_filter", "title": "Filter Array Column", "topics": [ "hof_filter" ] }, { "page": "hof_forall", "title": "Checks whether all elements in an array satisfy a predicate", "topics": [ "hof_forall" ] }, { "page": "hof_map_filter", "title": "Filters a map", "topics": [ "hof_map_filter" ] }, { "page": "hof_map_zip_with", "title": "Merges two maps into one", "topics": [ "hof_map_zip_with" ] }, { "page": "hof_transform", "title": "Transform Array Column", "topics": [ "hof_transform" ] }, { "page": "hof_transform_keys", "title": "Transforms keys of a map", "topics": [ "hof_transform_keys" ] }, { "page": "hof_transform_values", "title": "Transforms values of a map", "topics": [ "hof_transform_values" ] }, { "page": "hof_zip_with", "title": "Combines 2 Array Columns", "topics": [ "hof_zip_with" ] }, { "page": "inner_join", "title": "Inner join", "topics": [ "inner_join" ] }, { "page": "invoke", "title": "Invoke a Method on a JVM Object", "topics": [ "invoke", "invoke_new", "invoke_static" ] }, { "page": "j_invoke", "title": "Invoke a Java function.", "topics": [ "j_invoke", "j_invoke_new", "j_invoke_static" ] }, { "page": "jarray", "title": "Instantiate a Java array with a specific element type.", "topics": [ "jarray" ] }, { "page": "jfloat", "title": "Instantiate a Java float type.", "topics": [ "jfloat" ] }, { "page": "jfloat_array", "title": "Instantiate an Array[Float].", "topics": [ "jfloat_array" ] }, { "page": "join.tbl_spark", "title": "Join Spark tbls.", "topics": [ "full_join.tbl_spark", "inner_join.tbl_spark", "join.tbl_spark", "left_join.tbl_spark", "right_join.tbl_spark" ] }, { "page": "left_join", "title": "Left join", "topics": [ "left_join" ] }, { "page": "list_sparklyr_jars", "title": "list all sparklyr-*.jar files that have been built", "topics": [ "list_sparklyr_jars" ] }, { "page": "livy_config", "title": "Create a Spark Configuration for Livy", "topics": [ "livy_config" ] }, { "page": "livy_service", "title": "Start Livy", "topics": [ "livy_service_start", "livy_service_stop" ] }, { "page": "ml_aft_survival_regression", "title": "Spark ML - Survival Regression", "concept": [ "ml algorithms" ], "topics": [ "ml_aft_survival_regression", "ml_survival_regression" ] }, { "page": "ml_als", "title": "Spark ML - ALS", "topics": [ "ml_als", "ml_recommend" ] }, { "page": "ml_als_tidiers", "title": "Tidying methods for Spark ML ALS", "topics": [ "augment.ml_model_als", "glance.ml_model_als", "ml_als_tidiers", "tidy.ml_model_als" ] }, { "page": "ml_bisecting_kmeans", "title": "Spark ML - Bisecting K-Means Clustering", "topics": [ "ml_bisecting_kmeans" ] }, { "page": "ml_chisquare_test", "title": "Chi-square hypothesis testing for categorical data.", "topics": [ "ml_chisquare_test" ] }, { "page": "ml_clustering_evaluator", "title": "Spark ML - Clustering Evaluator", "topics": [ "ml_clustering_evaluator" ] }, { "page": "ml_corr", "title": "Compute correlation matrix", "topics": [ "ml_corr" ] }, { "page": "ml_decision_tree", "title": "Spark ML - Decision Trees", "concept": [ "ml algorithms" ], "topics": [ "ml_decision_tree", "ml_decision_tree_classifier", "ml_decision_tree_regressor" ] }, { "page": "ml_default_stop_words", "title": "Default stop words", "topics": [ "ml_default_stop_words" ] }, { "page": "ml_evaluate", "title": "Evaluate the Model on a Validation Set", "topics": [ "ml_evaluate", "ml_evaluate.ml_evaluator", "ml_evaluate.ml_generalized_linear_regression_model", "ml_evaluate.ml_linear_regression_model", "ml_evaluate.ml_logistic_regression_model", "ml_evaluate.ml_model_classification", "ml_evaluate.ml_model_clustering", "ml_evaluate.ml_model_generalized_linear_regression", "ml_evaluate.ml_model_linear_regression", "ml_evaluate.ml_model_logistic_regression" ] }, { "page": "ml_evaluator", "title": "Spark ML - Evaluators", "topics": [ "ml_binary_classification_eval", "ml_binary_classification_evaluator", "ml_classification_eval", "ml_evaluator", "ml_multiclass_classification_evaluator", "ml_regression_evaluator" ] }, { "page": "ml_feature_importances", "title": "Spark ML - Feature Importance for Tree Models", "topics": [ "ml_feature_importances", "ml_tree_feature_importance" ] }, { "page": "ml_fpgrowth", "title": "Frequent Pattern Mining - FPGrowth", "topics": [ "ml_association_rules", "ml_fpgrowth", "ml_freq_itemsets" ] }, { "page": "ml_gaussian_mixture", "title": "Spark ML - Gaussian Mixture clustering.", "topics": [ "ml_gaussian_mixture" ] }, { "page": "ml_gradient_boosted_trees", "title": "Spark ML - Gradient Boosted Trees", "concept": [ "ml algorithms" ], "topics": [ "ml_gbt_classifier", "ml_gbt_regressor", "ml_gradient_boosted_trees" ] }, { "page": "ml_generalized_linear_regression", "title": "Spark ML - Generalized Linear Regression", "concept": [ "ml algorithms" ], "topics": [ "ml_generalized_linear_regression" ] }, { "page": "ml_glm_tidiers", "title": "Tidying methods for Spark ML linear models", "topics": [ "augment.ml_model_generalized_linear_regression", "augment.ml_model_linear_regression", "augment._ml_model_linear_regression", "glance.ml_model_generalized_linear_regression", "glance.ml_model_linear_regression", "ml_glm_tidiers", "tidy.ml_model_generalized_linear_regression", "tidy.ml_model_linear_regression" ] }, { "page": "ml_isotonic_regression", "title": "Spark ML - Isotonic Regression", "concept": [ "ml algorithms" ], "topics": [ "ml_isotonic_regression" ] }, { "page": "ml_isotonic_regression_tidiers", "title": "Tidying methods for Spark ML Isotonic Regression", "topics": [ "augment.ml_model_isotonic_regression", "glance.ml_model_isotonic_regression", "ml_isotonic_regression_tidiers", "tidy.ml_model_isotonic_regression" ] }, { "page": "ml_kmeans", "title": "Spark ML - K-Means Clustering", "topics": [ "ml_compute_cost", "ml_compute_silhouette_measure", "ml_kmeans" ] }, { "page": "ml_kmeans_cluster_eval", "title": "Evaluate a K-mean clustering", "topics": [ "ml_kmeans_cluster_eval" ] }, { "page": "ml_lda", "title": "Spark ML - Latent Dirichlet Allocation", "topics": [ "ml_describe_topics", "ml_lda", "ml_log_likelihood", "ml_log_perplexity", "ml_topics_matrix" ] }, { "page": "ml_lda_tidiers", "title": "Tidying methods for Spark ML LDA models", "topics": [ "augment.ml_model_lda", "glance.ml_model_lda", "ml_lda_tidiers", "tidy.ml_model_lda" ] }, { "page": "ml_linear_regression", "title": "Spark ML - Linear Regression", "concept": [ "ml algorithms" ], "topics": [ "ml_linear_regression" ] }, { "page": "ml_linear_svc", "title": "Spark ML - LinearSVC", "concept": [ "ml algorithms" ], "topics": [ "ml_linear_svc" ] }, { "page": "ml_linear_svc_tidiers", "title": "Tidying methods for Spark ML linear svc", "topics": [ "augment.ml_model_linear_svc", "glance.ml_model_linear_svc", "ml_linear_svc_tidiers", "tidy.ml_model_linear_svc" ] }, { "page": "ml_logistic_regression", "title": "Spark ML - Logistic Regression", "concept": [ "ml algorithms" ], "topics": [ "ml_logistic_regression" ] }, { "page": "ml_logistic_regression_tidiers", "title": "Tidying methods for Spark ML Logistic Regression", "topics": [ "augment.ml_model_logistic_regression", "augment._ml_model_logistic_regression", "glance.ml_model_logistic_regression", "ml_logistic_regression_tidiers", "tidy.ml_model_logistic_regression" ] }, { "page": "ml_metrics_binary", "title": "Extracts metrics from a fitted table", "topics": [ "ml_metrics_binary" ] }, { "page": "ml_metrics_multiclass", "title": "Extracts metrics from a fitted table", "topics": [ "ml_metrics_multiclass" ] }, { "page": "ml_metrics_regression", "title": "Extracts metrics from a fitted table", "topics": [ "ml_metrics_regression" ] }, { "page": "ml_model_data", "title": "Extracts data associated with a Spark ML model", "topics": [ "ml_model_data" ] }, { "page": "ml_multilayer_perceptron_classifier", "title": "Spark ML - Multilayer Perceptron", "concept": [ "ml algorithms" ], "topics": [ "ml_multilayer_perceptron", "ml_multilayer_perceptron_classifier" ] }, { "page": "ml_multilayer_perceptron_tidiers", "title": "Tidying methods for Spark ML MLP", "topics": [ "augment.ml_model_multilayer_perceptron_classification", "glance.ml_model_multilayer_perceptron_classification", "ml_multilayer_perceptron_tidiers", "tidy.ml_model_multilayer_perceptron_classification" ] }, { "page": "ml_naive_bayes", "title": "Spark ML - Naive-Bayes", "concept": [ "ml algorithms" ], "topics": [ "ml_naive_bayes" ] }, { "page": "ml_naive_bayes_tidiers", "title": "Tidying methods for Spark ML Naive Bayes", "topics": [ "augment.ml_model_naive_bayes", "glance.ml_model_naive_bayes", "ml_naive_bayes_tidiers", "tidy.ml_model_naive_bayes" ] }, { "page": "ml_one_vs_rest", "title": "Spark ML - OneVsRest", "concept": [ "ml algorithms" ], "topics": [ "ml_one_vs_rest" ] }, { "page": "ml_pca_tidiers", "title": "Tidying methods for Spark ML Principal Component Analysis", "topics": [ "augment.ml_model_pca", "glance.ml_model_pca", "ml_pca_tidiers", "tidy.ml_model_pca" ] }, { "page": "ml_pipeline", "title": "Spark ML - Pipelines", "topics": [ "ml_pipeline" ] }, { "page": "ml_power_iteration", "title": "Spark ML - Power Iteration Clustering", "topics": [ "ml_power_iteration" ] }, { "page": "ml_prefixspan", "title": "Frequent Pattern Mining - PrefixSpan", "topics": [ "ml_freq_seq_patterns", "ml_prefixspan" ] }, { "page": "ml_random_forest", "title": "Spark ML - Random Forest", "concept": [ "ml algorithms" ], "topics": [ "ml_random_forest", "ml_random_forest_classifier", "ml_random_forest_regressor" ] }, { "page": "ml_stage", "title": "Spark ML - Pipeline stage extraction", "topics": [ "ml_stage", "ml_stages" ] }, { "page": "ml_summary", "title": "Spark ML - Extraction of summary metrics", "topics": [ "ml_summary" ] }, { "page": "ml_survival_regression_tidiers", "title": "Tidying methods for Spark ML Survival Regression", "topics": [ "augment.ml_model_aft_survival_regression", "glance.ml_model_aft_survival_regression", "ml_survival_regression_tidiers", "tidy.ml_model_aft_survival_regression" ] }, { "page": "ml_tree_tidiers", "title": "Tidying methods for Spark ML tree models", "topics": [ "augment.ml_model_decision_tree_classification", "augment.ml_model_decision_tree_regression", "augment.ml_model_gbt_classification", "augment.ml_model_gbt_regression", "augment.ml_model_random_forest_classification", "augment.ml_model_random_forest_regression", "augment._ml_model_decision_tree_classification", "augment._ml_model_decision_tree_regression", "augment._ml_model_gbt_classification", "augment._ml_model_gbt_regression", "augment._ml_model_random_forest_classification", "augment._ml_model_random_forest_regression", "glance.ml_model_decision_tree_classification", "glance.ml_model_decision_tree_regression", "glance.ml_model_gbt_classification", "glance.ml_model_gbt_regression", "glance.ml_model_random_forest_classification", "glance.ml_model_random_forest_regression", "ml_tree_tidiers", "tidy.ml_model_decision_tree_classification", "tidy.ml_model_decision_tree_regression", "tidy.ml_model_gbt_classification", "tidy.ml_model_gbt_regression", "tidy.ml_model_random_forest_classification", "tidy.ml_model_random_forest_regression" ] }, { "page": "ml_uid", "title": "Spark ML - UID", "topics": [ "ml_uid" ] }, { "page": "ml_unsupervised_tidiers", "title": "Tidying methods for Spark ML unsupervised models", "topics": [ "augment.ml_model_bisecting_kmeans", "augment.ml_model_gaussian_mixture", "augment.ml_model_kmeans", "glance.ml_model_bisecting_kmeans", "glance.ml_model_gaussian_mixture", "glance.ml_model_kmeans", "ml_unsupervised_tidiers", "tidy.ml_model_bisecting_kmeans", "tidy.ml_model_gaussian_mixture", "tidy.ml_model_kmeans" ] }, { "page": "ml-params", "title": "Spark ML - ML Params", "topics": [ "ml-params", "ml_is_set", "ml_param", "ml_params", "ml_param_map" ] }, { "page": "ml-persistence", "title": "Spark ML - Model Persistence", "topics": [ "ml-persistence", "ml_load", "ml_save", "ml_save.ml_model" ] }, { "page": "ml-transform-methods", "title": "Spark ML - Transform, fit, and predict methods (ml_ interface)", "topics": [ "is_ml_estimator", "is_ml_transformer", "ml-transform-methods", "ml_fit", "ml_fit.default", "ml_fit_and_transform", "ml_predict", "ml_predict.ml_model_classification", "ml_transform" ] }, { "page": "ml-tuning", "title": "Spark ML - Tuning", "topics": [ "ml-tuning", "ml_cross_validator", "ml_sub_models", "ml_train_validation_split", "ml_validation_metrics" ] }, { "page": "mutate", "title": "Mutate", "topics": [ "mutate" ] }, { "page": "na.replace", "title": "Replace Missing Values in Objects", "topics": [ "na.replace" ] }, { "page": "nest", "title": "Nest", "topics": [ "nest" ] }, { "page": "pivot_longer", "title": "Pivot longer", "topics": [ "pivot_longer" ] }, { "page": "pivot_wider", "title": "Pivot wider", "topics": [ "pivot_wider" ] }, { "page": "random_string", "title": "Random string generation", "topics": [ "random_string" ] }, { "page": "reactiveSpark", "title": "Reactive spark reader", "topics": [ "reactiveSpark" ] }, { "page": "register_extension", "title": "Register a Package that Implements a Spark Extension", "topics": [ "registered_extensions", "register_extension" ] }, { "page": "registerDoSpark", "title": "Register a Parallel Backend", "topics": [ "registerDoSpark" ] }, { "page": "replace_na", "title": "Replace NA", "topics": [ "replace_na" ] }, { "page": "right_join", "title": "Right join", "topics": [ "right_join" ] }, { "page": "sdf_along", "title": "Create DataFrame for along Object", "topics": [ "sdf_along" ] }, { "page": "sdf_bind", "title": "Bind multiple Spark DataFrames by row and column", "topics": [ "sdf_bind", "sdf_bind_cols", "sdf_bind_rows" ] }, { "page": "sdf_broadcast", "title": "Broadcast hint", "topics": [ "sdf_broadcast" ] }, { "page": "sdf_checkpoint", "title": "Checkpoint a Spark DataFrame", "topics": [ "sdf_checkpoint" ] }, { "page": "sdf_coalesce", "title": "Coalesces a Spark DataFrame", "topics": [ "sdf_coalesce" ] }, { "page": "sdf_collect", "title": "Collect a Spark DataFrame into R.", "topics": [ "sdf_collect" ] }, { "page": "sdf_copy_to", "title": "Copy an Object into Spark", "concept": [ "Spark data frames" ], "topics": [ "sdf_copy_to", "sdf_import" ] }, { "page": "sdf_crosstab", "title": "Cross Tabulation", "topics": [ "sdf_crosstab" ] }, { "page": "sdf_debug_string", "title": "Debug Info for Spark DataFrame", "topics": [ "sdf_debug_string" ] }, { "page": "sdf_describe", "title": "Compute summary statistics for columns of a data frame", "topics": [ "sdf_describe" ] }, { "page": "sdf_dim", "title": "Support for Dimension Operations", "topics": [ "sdf_dim", "sdf_ncol", "sdf_nrow" ] }, { "page": "sdf_distinct", "title": "Invoke distinct on a Spark DataFrame", "concept": [ "Spark data frames" ], "topics": [ "sdf_distinct" ] }, { "page": "sdf_drop_duplicates", "title": "Remove duplicates from a Spark DataFrame", "topics": [ "sdf_drop_duplicates" ] }, { "page": "sdf_expand_grid", "title": "Create a Spark dataframe containing all combinations of inputs", "topics": [ "sdf_expand_grid" ] }, { "page": "sdf_from_avro", "title": "Convert column(s) from avro format", "topics": [ "sdf_from_avro" ] }, { "page": "sdf_is_streaming", "title": "Spark DataFrame is Streaming", "topics": [ "sdf_is_streaming" ] }, { "page": "sdf_last_index", "title": "Returns the last index of a Spark DataFrame", "topics": [ "sdf_last_index" ] }, { "page": "sdf_len", "title": "Create DataFrame for Length", "topics": [ "sdf_len" ] }, { "page": "sdf_num_partitions", "title": "Gets number of partitions of a Spark DataFrame", "topics": [ "sdf_num_partitions" ] }, { "page": "sdf_partition_sizes", "title": "Compute the number of records within each partition of a Spark DataFrame", "topics": [ "sdf_partition_sizes" ] }, { "page": "sdf_persist", "title": "Persist a Spark DataFrame", "topics": [ "sdf_persist" ] }, { "page": "sdf_pivot", "title": "Pivot a Spark DataFrame", "topics": [ "sdf_pivot" ] }, { "page": "sdf_project", "title": "Project features onto principal components", "topics": [ "sdf_project" ] }, { "page": "sdf_quantile", "title": "Compute (Approximate) Quantiles with a Spark DataFrame", "topics": [ "sdf_quantile" ] }, { "page": "sdf_random_split", "title": "Partition a Spark Dataframe", "concept": [ "Spark data frames" ], "topics": [ "sdf_partition", "sdf_random_split" ] }, { "page": "sdf_rbeta", "title": "Generate random samples from a Beta distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rbeta" ] }, { "page": "sdf_rbinom", "title": "Generate random samples from a binomial distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rbinom" ] }, { "page": "sdf_rcauchy", "title": "Generate random samples from a Cauchy distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rcauchy" ] }, { "page": "sdf_rchisq", "title": "Generate random samples from a chi-squared distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rchisq" ] }, { "page": "sdf_read_column", "title": "Read a Column from a Spark DataFrame", "topics": [ "sdf_read_column" ] }, { "page": "sdf_register", "title": "Register a Spark DataFrame", "concept": [ "Spark data frames" ], "topics": [ "sdf_register" ] }, { "page": "sdf_repartition", "title": "Repartition a Spark DataFrame", "topics": [ "sdf_repartition" ] }, { "page": "sdf_residuals", "title": "Model Residuals", "topics": [ "sdf_residuals", "sdf_residuals.ml_model_generalized_linear_regression", "sdf_residuals.ml_model_linear_regression" ] }, { "page": "sdf_rexp", "title": "Generate random samples from an exponential distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rexp" ] }, { "page": "sdf_rgamma", "title": "Generate random samples from a Gamma distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rgamma" ] }, { "page": "sdf_rgeom", "title": "Generate random samples from a geometric distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rgeom" ] }, { "page": "sdf_rhyper", "title": "Generate random samples from a hypergeometric distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rhyper" ] }, { "page": "sdf_rlnorm", "title": "Generate random samples from a log normal distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rlnorm" ] }, { "page": "sdf_rnorm", "title": "Generate random samples from the standard normal distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rnorm" ] }, { "page": "sdf_rpois", "title": "Generate random samples from a Poisson distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rpois" ] }, { "page": "sdf_rt", "title": "Generate random samples from a t-distribution", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rt" ] }, { "page": "sdf_runif", "title": "Generate random samples from the uniform distribution U(0, 1).", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_runif" ] }, { "page": "sdf_rweibull", "title": "Generate random samples from a Weibull distribution.", "concept": [ "Spark statistical routines" ], "topics": [ "sdf_rweibull" ] }, { "page": "sdf_sample", "title": "Randomly Sample Rows from a Spark DataFrame", "concept": [ "Spark data frames" ], "topics": [ "sdf_sample" ] }, { "page": "sdf_schema", "title": "Read the Schema of a Spark DataFrame", "topics": [ "sdf_schema" ] }, { "page": "sdf_separate_column", "title": "Separate a Vector Column into Scalar Columns", "topics": [ "sdf_separate_column" ] }, { "page": "sdf_seq", "title": "Create DataFrame for Range", "topics": [ "sdf_seq" ] }, { "page": "sdf_sort", "title": "Sort a Spark DataFrame", "concept": [ "Spark data frames" ], "topics": [ "sdf_sort" ] }, { "page": "sdf_sql", "title": "Spark DataFrame from SQL", "topics": [ "sdf_sql" ] }, { "page": "sdf_to_avro", "title": "Convert column(s) to avro format", "topics": [ "sdf_to_avro" ] }, { "page": "sdf_unnest_longer", "title": "Unnest longer", "topics": [ "sdf_unnest_longer" ] }, { "page": "sdf_unnest_wider", "title": "Unnest wider", "topics": [ "sdf_unnest_wider" ] }, { "page": "sdf_weighted_sample", "title": "Perform Weighted Random Sampling on a Spark DataFrame", "concept": [ "Spark data frames" ], "topics": [ "sdf_weighted_sample" ] }, { "page": "sdf_with_sequential_id", "title": "Add a Sequential ID Column to a Spark DataFrame", "topics": [ "sdf_with_sequential_id" ] }, { "page": "sdf_with_unique_id", "title": "Add a Unique ID Column to a Spark DataFrame", "topics": [ "sdf_with_unique_id" ] }, { "page": "sdf-saveload", "title": "Save / Load a Spark DataFrame", "topics": [ "sdf-saveload", "sdf_load_parquet", "sdf_load_table", "sdf_save_parquet", "sdf_save_table" ] }, { "page": "sdf-transform-methods", "title": "Spark ML - Transform, fit, and predict methods (sdf_ interface)", "topics": [ "sdf-transform-methods", "sdf_fit", "sdf_fit_and_transform", "sdf_predict", "sdf_transform" ] }, { "page": "select", "title": "Select", "topics": [ "select" ] }, { "page": "separate", "title": "Separate", "topics": [ "separate" ] }, { "page": "spark_adaptive_query_execution", "title": "Retrieves or sets status of Spark AQE", "concept": [ "Spark runtime configuration" ], "topics": [ "spark_adaptive_query_execution" ] }, { "page": "spark_advisory_shuffle_partition_size", "title": "Retrieves or sets advisory size of the shuffle partition", "concept": [ "Spark runtime configuration" ], "topics": [ "spark_advisory_shuffle_partition_size" ] }, { "page": "spark_apply", "title": "Apply an R Function in Spark", "topics": [ "spark_apply" ] }, { "page": "spark_apply_bundle", "title": "Create Bundle for Spark Apply", "topics": [ "spark_apply_bundle" ] }, { "page": "spark_apply_log", "title": "Log Writer for Spark Apply", "topics": [ "spark_apply_log" ] }, { "page": "spark_auto_broadcast_join_threshold", "title": "Retrieves or sets the auto broadcast join threshold", "concept": [ "Spark runtime configuration" ], "topics": [ "spark_auto_broadcast_join_threshold" ] }, { "page": "spark_coalesce_initial_num_partitions", "title": "Retrieves or sets initial number of shuffle partitions before coalescing", "concept": [ "Spark runtime configuration" ], "topics": [ "spark_coalesce_initial_num_partitions" ] }, { "page": "spark_coalesce_min_num_partitions", "title": "Retrieves or sets the minimum number of shuffle partitions after coalescing", "concept": [ "Spark runtime configuration" ], "topics": [ "spark_coalesce_min_num_partitions" ] }, { "page": "spark_coalesce_shuffle_partitions", "title": "Retrieves or sets whether coalescing contiguous shuffle partitions is enabled", "concept": [ "Spark runtime configuration" ], "topics": [ "spark_coalesce_shuffle_partitions" ] }, { "page": "spark_compilation_spec", "title": "Define a Spark Compilation Specification", "topics": [ "spark_compilation_spec" ] }, { "page": "spark_config", "title": "Read Spark Configuration", "topics": [ "spark_config" ] }, { "page": "spark_config_kubernetes", "title": "Kubernetes Configuration", "topics": [ "spark_config_kubernetes" ] }, { "page": "spark_config_settings", "title": "Retrieve Available Settings", "topics": [ "spark_config_settings" ] }, { "page": "spark_connect_method", "title": "Function that negotiates the connection with the Spark back-end", "topics": [ "spark_connect_method" ] }, { "page": "spark_connection", "title": "Retrieve the Spark Connection Associated with an R Object", "topics": [ "spark_connection" ] }, { "page": "spark_connection_find", "title": "Find Spark Connection", "topics": [ "spark_connection_find" ] }, { "page": "spark_connection-class", "title": "spark_connection class", "topics": [ "spark_connection-class" ] }, { "page": "spark_context_config", "title": "Runtime configuration interface for the Spark Context.", "topics": [ "spark_context_config" ] }, { "page": "spark_dataframe", "title": "Retrieve a Spark DataFrame", "topics": [ "spark_dataframe" ] }, { "page": "spark_default_compilation_spec", "title": "Default Compilation Specification for Spark Extensions", "topics": [ "spark_default_compilation_spec" ] }, { "page": "spark_dependency", "title": "Define a Spark dependency", "topics": [ "spark_dependency" ] }, { "page": "spark_dependency_fallback", "title": "Fallback to Spark Dependency", "topics": [ "spark_dependency_fallback" ] }, { "page": "spark_extension", "title": "Create Spark Extension", "topics": [ "spark_extension" ] }, { "page": "spark_home_set", "title": "Set the SPARK_HOME environment variable", "topics": [ "spark_home_set" ] }, { "page": "spark_ide_connection_open", "title": "Set of functions to provide integration with the RStudio IDE", "topics": [ "spark_ide_columns", "spark_ide_connection_actions", "spark_ide_connection_closed", "spark_ide_connection_open", "spark_ide_connection_updated", "spark_ide_objects", "spark_ide_preview" ] }, { "page": "spark_insert_table", "title": "Inserts a Spark DataFrame into a Spark table", "concept": [ "Spark serialization routines" ], "topics": [ "spark_insert_table" ] }, { "page": "spark_install", "title": "Download and install various versions of Spark", "topics": [ "spark_available_versions", "spark_install", "spark_installed_versions", "spark_install_dir", "spark_install_tar", "spark_uninstall" ] }, { "page": "spark_integ_test_skip", "title": "It lets the package know if it should test a particular functionality or not", "topics": [ "spark_integ_test_skip" ] }, { "page": "spark_jobj", "title": "Retrieve a Spark JVM Object Reference", "topics": [ "spark_jobj" ] }, { "page": "spark_jobj-class", "title": "spark_jobj class", "topics": [ "spark_jobj-class" ] }, { "page": "spark_last_error", "title": "Surfaces the last error from Spark captured by internal `spark_error` function", "topics": [ "spark_last_error" ] }, { "page": "spark_load_table", "title": "Reads from a Spark Table into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_load_table" ] }, { "page": "spark_log", "title": "View Entries in the Spark Log", "topics": [ "spark_log" ] }, { "page": "spark_read", "title": "Read file(s) into a Spark DataFrame using a custom reader", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read" ] }, { "page": "spark_read_avro", "title": "Read Apache Avro data into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_avro" ] }, { "page": "spark_read_binary", "title": "Read binary data into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_binary" ] }, { "page": "spark_read_csv", "title": "Read a CSV file into a Spark DataFrame", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_csv" ] }, { "page": "spark_read_delta", "title": "Read from Delta Lake into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_delta" ] }, { "page": "spark_read_image", "title": "Read image data into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_image" ] }, { "page": "spark_read_jdbc", "title": "Read from JDBC connection into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_jdbc" ] }, { "page": "spark_read_json", "title": "Read a JSON file into a Spark DataFrame", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_json" ] }, { "page": "spark_read_libsvm", "title": "Read libsvm file into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_libsvm" ] }, { "page": "spark_read_orc", "title": "Read a ORC file into a Spark DataFrame", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_orc" ] }, { "page": "spark_read_parquet", "title": "Read a Parquet file into a Spark DataFrame", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_parquet" ] }, { "page": "spark_read_source", "title": "Read from a generic source into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_source" ] }, { "page": "spark_read_table", "title": "Reads from a Spark Table into a Spark DataFrame.", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_table" ] }, { "page": "spark_read_text", "title": "Read a Text file into a Spark DataFrame", "concept": [ "Spark serialization routines" ], "topics": [ "spark_read_text" ] }, { "page": "spark_save_table", "title": "Saves a Spark DataFrame as a Spark table", "concept": [ "Spark serialization routines" ], "topics": [ "spark_save_table" ] }, { "page": "spark_configuration", "title": "Runtime configuration interface for the Spark Session", "concept": [ "Spark runtime configuration" ], "topics": [ "spark_session_config" ] }, { "page": "spark_statistical_routines", "title": "Generate random samples from some distribution", "topics": [ "spark_statistical_routines" ] }, { "page": "spark_table_name", "title": "Generate a Table Name from Expression", "topics": [ "spark_table_name" ] }, { "page": "spark_version", "title": "Get the Spark Version Associated with a Spark Connection", "topics": [ "spark_version" ] }, { "page": "spark_version_from_home", "title": "Get the Spark Version Associated with a Spark Installation", "topics": [ "spark_version_from_home" ] }, { "page": "spark_web", "title": "Open the Spark web interface", "topics": [ "spark_web" ] }, { "page": "spark_write", "title": "Write Spark DataFrame to file using a custom writer", "topics": [ "spark_write" ] }, { "page": "spark_write_avro", "title": "Serialize a Spark DataFrame into Apache Avro format", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_avro" ] }, { "page": "spark_write_csv", "title": "Write a Spark DataFrame to a CSV", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_csv" ] }, { "page": "spark_write_delta", "title": "Writes a Spark DataFrame into Delta Lake", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_delta" ] }, { "page": "spark_write_jdbc", "title": "Writes a Spark DataFrame into a JDBC table", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_jdbc" ] }, { "page": "spark_write_json", "title": "Write a Spark DataFrame to a JSON file", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_json" ] }, { "page": "spark_write_orc", "title": "Write a Spark DataFrame to a ORC file", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_orc" ] }, { "page": "spark_write_parquet", "title": "Write a Spark DataFrame to a Parquet file", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_parquet" ] }, { "page": "spark_write_rds", "title": "Write Spark DataFrame to RDS files", "topics": [ "spark_write_rds" ] }, { "page": "spark_write_source", "title": "Writes a Spark DataFrame into a generic source", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_source" ] }, { "page": "spark_write_table", "title": "Writes a Spark DataFrame into a Spark table", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_table" ] }, { "page": "spark_write_text", "title": "Write a Spark DataFrame to a Text file", "concept": [ "Spark serialization routines" ], "topics": [ "spark_write_text" ] }, { "page": "spark-api", "title": "Access the Spark API", "topics": [ "hive_context", "java_context", "spark-api", "spark_context", "spark_session" ] }, { "page": "spark-connections", "title": "Manage Spark Connections", "topics": [ "spark-connections", "spark_connect", "spark_connection_is_open", "spark_disconnect", "spark_disconnect_all", "spark_submit" ] }, { "page": "sparklyr_get_backend_port", "title": "Return the port number of a `sparklyr` backend.", "topics": [ "sparklyr_get_backend_port" ] }, { "page": "src_databases", "title": "Show database list", "topics": [ "src_databases" ] }, { "page": "stream_find", "title": "Find Stream", "topics": [ "stream_find" ] }, { "page": "stream_generate_test", "title": "Generate Test Stream", "topics": [ "stream_generate_test" ] }, { "page": "stream_id", "title": "Spark Stream's Identifier", "topics": [ "stream_id" ] }, { "page": "stream_lag", "title": "Apply lag function to columns of a Spark Streaming DataFrame", "topics": [ "stream_lag" ] }, { "page": "stream_name", "title": "Spark Stream's Name", "topics": [ "stream_name" ] }, { "page": "stream_read_csv", "title": "Read files created by the stream", "topics": [ "stream_read_cloudfiles", "stream_read_csv", "stream_read_delta", "stream_read_json", "stream_read_kafka", "stream_read_orc", "stream_read_parquet", "stream_read_socket", "stream_read_table", "stream_read_text" ] }, { "page": "stream_render", "title": "Render Stream", "topics": [ "stream_render" ] }, { "page": "stream_stats", "title": "Stream Statistics", "topics": [ "stream_stats" ] }, { "page": "stream_stop", "title": "Stops a Spark Stream", "topics": [ "stream_stop" ] }, { "page": "stream_trigger_continuous", "title": "Spark Stream Continuous Trigger", "topics": [ "stream_trigger_continuous" ] }, { "page": "stream_trigger_interval", "title": "Spark Stream Interval Trigger", "topics": [ "stream_trigger_interval" ] }, { "page": "stream_view", "title": "View Stream", "topics": [ "stream_view" ] }, { "page": "stream_watermark", "title": "Watermark Stream", "topics": [ "stream_watermark" ] }, { "page": "stream_write_csv", "title": "Write files to the stream", "concept": [ "Spark stream serialization" ], "topics": [ "stream_write_console", "stream_write_csv", "stream_write_delta", "stream_write_json", "stream_write_kafka", "stream_write_orc", "stream_write_parquet", "stream_write_text" ] }, { "page": "stream_write_memory", "title": "Write Memory Stream", "concept": [ "Spark stream serialization" ], "topics": [ "stream_write_memory" ] }, { "page": "stream_write_table", "title": "Write Stream to Table", "concept": [ "Spark stream serialization" ], "topics": [ "stream_write_table" ] }, { "page": "tbl_cache", "title": "Cache a Spark Table", "topics": [ "tbl_cache" ] }, { "page": "tbl_change_db", "title": "Use specific database", "topics": [ "tbl_change_db" ] }, { "page": "tbl_uncache", "title": "Uncache a Spark Table", "topics": [ "tbl_uncache" ] }, { "page": "transform_sdf", "title": "transform a subset of column(s) in a Spark Dataframe", "topics": [ "transform_sdf" ] }, { "page": "tune_grid_spark", "title": "Perform Tidymodels grid search tuning inside Spark", "topics": [ "tune_grid_spark" ] }, { "page": "unite", "title": "Unite", "topics": [ "unite" ] }, { "page": "unnest", "title": "Unnest", "topics": [ "unnest" ] } ], "_readme": "https://github.com/sparklyr/sparklyr/raw/HEAD/README.md", "_rundeps": [ "askpass", "blob", "cli", "codetools", "config", "cpp11", "curl", "DBI", "dbplyr", "dplyr", "generics", "globals", "glue", "httr", "jsonlite", "lifecycle", "magrittr", "mime", "openssl", "pillar", "pkgconfig", "purrr", "R6", "rlang", "rstudioapi", "stringi", "stringr", "sys", "tibble", "tidyr", "tidyselect", "utf8", "uuid", "vctrs", "withr", "xml2", "yaml" ], "_score": 15.167792847199959, "_indexed": true, "_nocasepkg": "sparklyr", "_universes": [ "sparklyr", "edgararuiz-zz" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "1.9.4", "date": "2026-05-17T09:35:04.000Z", "distro": "noble", "commit": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "fileid": "5318fad871ae88469a4f86221c2f4610d08ad09a0ddb22a378a496258ef8e6a4", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290" }, { "r": "4.6.0", "os": "linux", "version": "1.9.4", "date": "2026-05-17T09:35:02.000Z", "distro": "noble", "commit": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "fileid": "328023417af88711cd751d765b3fcb5bd53bcb87364f1a1bd91ab9fda31dbae5", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290" }, { "r": "4.5.3", "os": "mac", "version": "1.9.4", "date": "2026-05-17T09:34:48.000Z", "commit": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "fileid": "1951af01371de2b6591dbfd1586cc3a2878011e514789ac27d620a618d392231", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290" }, { "r": "4.6.0", "os": "mac", "version": "1.9.4", "date": "2026-05-17T09:34:59.000Z", "commit": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "fileid": "44cfed69ac9dc55d2058186024c61ced81200e337d134a1824a13bf342a94d7d", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290" }, { "r": "4.7.0", "os": "win", "version": "1.9.4", "date": "2026-05-17T09:34:05.000Z", "commit": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "fileid": "840d1c3a1ee959f03f5da5179558a7d7abde7ab7533e9527a96e29760bcfafc1", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290" }, { "r": "4.5.3", "os": "win", "version": "1.9.4", "date": "2026-05-17T09:33:56.000Z", "commit": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "fileid": "9c9f1672ed4b764f3f4434e2f5ab4703e55da20f25f9b15dcbad85c7b18e6b95", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290" }, { "r": "4.6.0", "os": "win", "version": "1.9.4", "date": "2026-05-17T09:34:08.000Z", "commit": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "fileid": "b4f0d34a3db3c8d04f111f03accb24e03920ab4a5ccf77bd86bc5a8ab9d1cd7f", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290" }, { "r": "4.6.0", "os": "wasm", "version": "1.9.4", "date": "2026-06-02T17:01:02.000Z", "commit": "704d894f70a0599dfa638f64a0eb502c4e8f4715", "fileid": "b30f0c9936ab496fcd64e0cef085dfc6d6d34e90beb0fae9873fa0145e9e36dd", "status": "success", "buildurl": "https://github.com/r-universe/sparklyr/actions/runs/25987104290" } ] }