{"id":"https://openalex.org/W4392120885","doi":"https://doi.org/10.48550/arxiv.2402.14528","title":"ACE : Off-Policy Actor-Critic with Causality-Aware Entropy Regularization","display_name":"ACE : Off-Policy Actor-Critic with Causality-Aware Entropy Regularization","publication_year":2024,"publication_date":"2024-02-22","ids":{"openalex":"https://openalex.org/W4392120885","doi":"https://doi.org/10.48550/arxiv.2402.14528"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.14528","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14528","pdf_url":"https://arxiv.org/pdf/2402.14528","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.14528","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079511023","display_name":"Tianying Ji","orcid":"https://orcid.org/0009-0006-8949-5880"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ji, Tianying","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005812409","display_name":"Yongyuan Liang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Yongyuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102707661","display_name":"Yan Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Yan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100440778","display_name":"Yu Luo","orcid":"https://orcid.org/0000-0001-6229-4639"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Yu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069392566","display_name":"Guowei Xu","orcid":"https://orcid.org/0000-0002-8941-9953"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Guowei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100562189","display_name":"Jiawei Guo","orcid":"https://orcid.org/0009-0006-7823-2508"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Jiawei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013554787","display_name":"Ruijie Zheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Ruijie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037162168","display_name":"Furong Huang","orcid":"https://orcid.org/0000-0002-7393-1047"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Furong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055546056","display_name":"Fuchun Sun","orcid":"https://orcid.org/0000-0003-3546-6305"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Fuchun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5049093671","display_name":"Huazhe Xu","orcid":"https://orcid.org/0000-0001-8578-1261"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Huazhe","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5079511023"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9695000052452087,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9695000052452087,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10400","display_name":"Network Security and Intrusion Detection","score":0.9591000080108643,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.9570000171661377,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/causality","display_name":"Causality (physics)","score":0.6644618511199951},{"id":"https://openalex.org/keywords/regularization","display_name":"Regularization (linguistics)","score":0.5990632772445679},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.5217388868331909},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.4898936152458191},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.4274718761444092},{"id":"https://openalex.org/keywords/mathematical-economics","display_name":"Mathematical economics","score":0.32089298963546753},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.28118813037872314},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2033444046974182},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.1584395468235016},{"id":"https://openalex.org/keywords/thermodynamics","display_name":"Thermodynamics","score":0.11959627270698547}],"concepts":[{"id":"https://openalex.org/C64357122","wikidata":"https://www.wikidata.org/wiki/Q1149766","display_name":"Causality (physics)","level":2,"score":0.6644618511199951},{"id":"https://openalex.org/C2776135515","wikidata":"https://www.wikidata.org/wiki/Q17143721","display_name":"Regularization (linguistics)","level":2,"score":0.5990632772445679},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.5217388868331909},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4898936152458191},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.4274718761444092},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.32089298963546753},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.28118813037872314},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2033444046974182},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.1584395468235016},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.11959627270698547},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.14528","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14528","pdf_url":"https://arxiv.org/pdf/2402.14528","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.14528","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.14528","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.14528","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14528","pdf_url":"https://arxiv.org/pdf/2402.14528","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2081494945","https://openalex.org/W2389053294","https://openalex.org/W1970893504","https://openalex.org/W4312071518","https://openalex.org/W2901208600","https://openalex.org/W1677090476","https://openalex.org/W830525666","https://openalex.org/W2330229995","https://openalex.org/W3014724311","https://openalex.org/W167306138"],"abstract_inverted_index":{"The":[0],"varying":[1],"significance":[2,37],"of":[3,38,89,130],"distinct":[4],"primitive":[5,40,70],"behaviors":[6,41],"during":[7,42],"the":[8,25,36,74,87,123],"policy":[9],"learning":[10],"process":[11],"has":[12],"been":[13],"overlooked":[14],"by":[15],"prior":[16],"model-free":[17,118],"RL":[18,119],"algorithms.":[19],"Leveraging":[20],"this":[21],"insight,":[22],"we":[23,72],"explore":[24],"causal":[26],"relationship":[27],"between":[28],"different":[29],"action":[30],"dimensions":[31],"and":[32,53,78,126,135],"rewards":[33],"to":[34,64,84,117],"evaluate":[35],"various":[39],"training.":[43],"We":[44],"introduce":[45,79],"a":[46,80,103],"causality-aware":[47],"entropy":[48],"term":[49],"that":[50],"effectively":[51],"identifies":[52],"prioritizes":[54],"actions":[55],"with":[56,98],"high":[57],"potential":[58],"impacts":[59],"for":[60],"efficient":[61,127],"exploration.":[62],"Furthermore,":[63],"prevent":[65],"excessive":[66],"focus":[67],"on":[68],"specific":[69],"behaviors,":[71],"analyze":[73],"gradient":[75],"dormancy":[76],"phenomenon":[77],"dormancy-guided":[81],"reset":[82],"mechanism":[83],"further":[85],"enhance":[86],"efficacy":[88],"our":[90,131],"method.":[91],"Our":[92],"proposed":[93],"algorithm,":[94],"ACE:":[95],"Off-policy":[96],"Actor-critic":[97],"Causality-aware":[99],"Entropy":[100],"regularization,":[101],"demonstrates":[102],"substantial":[104],"performance":[105],"advantage":[106],"across":[107],"29":[108],"diverse":[109],"continuous":[110],"control":[111],"tasks":[112],"spanning":[113],"7":[114],"domains":[115],"compared":[116],"baselines,":[120],"which":[121],"underscores":[122],"effectiveness,":[124],"versatility,":[125],"sample":[128],"efficiency":[129],"approach.":[132],"Benchmark":[133],"results":[134],"videos":[136],"are":[137],"available":[138],"at":[139],"https://ace-rl.github.io/.":[140]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}