.1 200 OK date: Wed, 01 Jul 2026 15:56:14 GMT server: mw-web.codfw.main-95db66ff4-cdpkg x-content-type-options: nosniff content-language: en accept-ch: reporting-endpoints: csp-report-to-endpoint='/w/api.php?action=cspreport&format=json'; content-security-policy: script-src 'unsafe-eval' blob: 'self' meta.wikimedia.org *.wikimedia.org *.wikipedia.org *.wikinews.org *.wiktionary.org *.wikibooks.org *.wikiversity.org *.wikisource.org wikisource.org *.wikiquote.org *.wikidata.org *.wikifunctions.org *.wikivoyage.org *.mediawiki.org mediawiki.org wikimedia.org *.wmflabs.org *.wmcloud.org *.toolforge.org wss://*.toolforge.org *.jsdelivr.net unpkg.com cdnjs.cloudflare.com raw.githubusercontent.com *.github.com code.jquery.com cdn.mathjax.org use.typekit.net fonts.cdnfonts.com use.fontawesome.com i.ytimg.com rsms.me doi.org localhost https://localhost:* http://localhost:* wss://localhost:* ws://localhost:* *.google.com *.gstatic.com *.googleapis.com *.translate.yandex.net yastatic.net ya.ru radically.github.io cdn.sammdot.ca cdn.fontshare.com viaf.org publicai-proxy.alaexis.workers.dev iiif.archive.org api.flickr.com live.staticflickr.com api.anthropic.com api.openai.com api.publicai.co catalogo.pusc.it parsifal.urbe.it opac.sbn.it overpass-api.de api.openrouteservice.org archive.org *.openstreetmap.org *.waymarkedtrails.org *.thunderforest.com registry.ipe.wiki analytics.ipe.wiki qlever.dev app.goacoustic.com wikipedia-archive.ourworldindata.org api.inaturalist.org inaturalist-open-data.s3.amazonaws.com validator.w3.org db.onlinewebfonts.com fontlibrary.org 'unsafe-inline' auth.wikimedia.org; default-src 'self' data: blob: upload.wikimedia.org https://commons.wikimedia.org meta.wikimedia.org *.wikimedia.org *.wikipedia.org *.wikinews.org *.wiktionary.org *.wikibooks.org *.wikiversity.org *.wikisource.org wikisource.org *.wikiquote.org *.wikidata.org *.wikifunctions.org *.wikivoyage.org *.mediawiki.org mediawiki.org wikimedia.org *.wmflabs.org *.wmcloud.org *.toolforge.org wss://*.toolforge.org *.jsdelivr.net unpkg.com cdnjs.cloudflare.com raw.githubusercontent.com *.github.com code.jquery.com cdn.mathjax.org use.typekit.net fonts.cdnfonts.com use.fontawesome.com i.ytimg.com rsms.me doi.org localhost https://localhost:* http://localhost:* wss://localhost:* ws://localhost:* *.google.com *.gstatic.com *.googleapis.com *.translate.yandex.net yastatic.net ya.ru radically.github.io cdn.sammdot.ca cdn.fontshare.com viaf.org publicai-proxy.alaexis.workers.dev iiif.archive.org api.flickr.com live.staticflickr.com api.anthropic.com api.openai.com api.publicai.co catalogo.pusc.it parsifal.urbe.it opac.sbn.it overpass-api.de api.openrouteservice.org archive.org *.openstreetmap.org *.waymarkedtrails.org *.thunderforest.com registry.ipe.wiki analytics.ipe.wiki qlever.dev app.goacoustic.com wikipedia-archive.ourworldindata.org api.inaturalist.org inaturalist-open-data.s3.amazonaws.com validator.w3.org db.onlinewebfonts.com fontlibrary.org en.wikibooks.org en.wikinews.org en.wikiquote.org en.wikisource.org en.wikiversity.org en.wikivoyage.org en.wiktionary.org www.mediawiki.org commons.wikimedia.org foundation.wikimedia.org incubator.wikimedia.org species.wikimedia.org wikimania.wikimedia.org www.wikidata.org www.wikifunctions.org auth.wikimedia.org; style-src 'self' data: blob: upload.wikimedia.org https://commons.wikimedia.org meta.wikimedia.org *.wikimedia.org *.wikipedia.org *.wikinews.org *.wiktionary.org *.wikibooks.org *.wikiversity.org *.wikisource.org wikisource.org *.wikiquote.org *.wikidata.org *.wikifunctions.org *.wikivoyage.org *.mediawiki.org mediawiki.org wikimedia.org *.wmflabs.org *.wmcloud.org *.toolforge.org wss://*.toolforge.org *.jsdelivr.net unpkg.com cdnjs.cloudflare.com raw.githubusercontent.com *.github.com code.jquery.com cdn.mathjax.org use.typekit.net fonts.cdnfonts.com use.fontawesome.com i.ytimg.com rsms.me doi.org localhost https://localhost:* http://localhost:* wss://localhost:* ws://localhost:* *.google.com *.gstatic.com *.googleapis.com *.translate.yandex.net yastatic.net ya.ru radically.github.io cdn.sammdot.ca cdn.fontshare.com viaf.org publicai-proxy.alaexis.workers.dev iiif.archive.org api.flickr.com live.staticflickr.com api.anthropic.com api.openai.com api.publicai.co catalogo.pusc.it parsifal.urbe.it opac.sbn.it overpass-api.de api.openrouteservice.org archive.org *.openstreetmap.org *.waymarkedtrails.org *.thunderforest.com registry.ipe.wiki analytics.ipe.wiki qlever.dev app.goacoustic.com wikipedia-archive.ourworldindata.org api.inaturalist.org inaturalist-open-data.s3.amazonaws.com validator.w3.org db.onlinewebfonts.com fontlibrary.org 'unsafe-inline'; object-src 'none'; report-uri /w/api.php?action=cspreport&format=json; report-to csp-report-to-endpoint last-modified: Tue, 30 Jun 2026 20:53:20 GMT content-type: text/html; charset=UTF-8 age: 76069 accept-ranges: bytes x-cache: cp2047 miss, cp2047 hit/1 x-cache-status: hit-front server-timing: cache;desc="hit-front", host;desc="cp2047" strict-transport-security: max-age=106384710; includeSubDomains; preload report-to: { "group": "wm_nel", "max_age": 604800, "endpoints": [{ "url": "https://intake-logging.wikimedia.org/v1/events?stream=w3c.reportingapi.network_error&schema_uri=/w3c/reportingapi/network_error/1.0.0" }] } nel: { "report_to": "wm_nel", "max_age": 604800, "failure_fraction": 0.05, "success_fraction": 0.0} set-cookie: WMF-Last-Access=02-Jul-2026;Path=/;HttpOnly;secure;Expires=Mon, 03 Aug 2026 12:00:00 GMT set-cookie: WMF-Last-Access-Global=02-Jul-2026;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Mon, 03 Aug 2026 12:00:00 GMT set-cookie: WMF-DP=7c2;Path=/;HttpOnly;secure;Expires=Fri, 03 Jul 2026 00:00:00 GMT x-client-ip: 75.28.18.94 cache-control: private, s-maxage=0, max-age=0, must-revalidate, no-transform vary: Accept-Encoding,X-Subdomain,Cookie,Authorization,User-Agent set-cookie: GeoIP=US:TX:McKinney:33.20:-96.62:v4; Path=/; secure; Domain=.wikipedia.org set-cookie: NetworkProbeLimit=0.001;Path=/;Secure;SameSite=None;Max-Age=3600 set-cookie: WMF-Uniq=FqkDwHxBrH-x8w3XNM9j0AORAAAAAFvdSNmj378u3akDlEm8bwrLgxpx8WFIahj8;Domain=.wikipedia.org;Path=/;HttpOnly;secure;SameSite=None;Expires=Fri, 02 Jul 2027 00:00:00 GMT content-length: 727443 x-request-id: 5a585c57-6c8a-4c16-a253-b582677e9132 x-analytics: Adversarial machine learning - Wikipedia Jump to content Main menu Main menu move to sidebar hide Navigation Main page Contents Current events Random article About Wikipedia Contact us Contribute Help Learn to edit Community portal Recent changes Upload file Special pages Search Search Appearance Donate Create account Log in Personal tools Donate Create account Log in Contents move to sidebar hide (Top) 1 History Toggle History subsection 1.1 Examples 2 Challenges in applying machine learning to security Toggle Challenges in applying machine learning to security subsection 2.1 Data collection issues 2.2 Labeling and ground truth challenges 2.3 Concept drift 2.4 Feature robustness 2.5 Class imbalance 2.6 System design and learning 2.7 Performance evaluation 2.8 Deployment and operation 3 Attack modalities Toggle Attack modalities subsection 3.1 Taxonomy 3.2 Strategies 3.2.1 Data poisoning 3.2.2 Byzantine attacks 3.2.3 Evasion 3.2.4 Model extraction 4 Categories Toggle Categories subsection 4.1 Adversarial attacks and training in linear models 4.2 Adversarial deep reinforcement learning 4.3 Adversarial natural language processing 5 Specific attack types Toggle Specific attack types subsection 5.1 Adversarial examples 5.1.1 Black box attacks 5.1.1.1 Simple black-box adversarial attacks 5.1.1.2 Square attack 5.1.1.3 HopSkipJump attack 5.1.2 White box attacks 5.1.2.1 Fast gradient sign method 5.1.2.2 Carlini & Wagner 6 Defenses Toggle Defenses subsection 6.1 Mechanisms 7 See also 8 References 9 External links Toggle the table of contents Adversarial machine learning 14 languages العربية Čeština Ελληνικά Español فارسی Bahasa Indonesia Italiano 한국어 Polski Português Simple English Українська 粵語 中文 Edit links Article Talk English Read Edit View history Tools Tools move to sidebar hide Actions Read Edit View history General What links here Related changes Upload file Permanent link Page information Cite this page Get shortened URL Switch to legacy parser Print/export Download as PDF Printable version In other projects Wikidata item Appearance move to sidebar hide From Wikipedia, the free encyclopedia Research field that lies at the intersection of machine learning and computer security Not to be confused with Generative adversarial network or Competitive learning . Part of a series on Machine learning and data mining Paradigms Supervised learning Unsupervised learning Semi-supervised learning Self-supervised learning Reinforcement learning Meta-learning Online learning Batch learning Curriculum learning Rule-based learning Neuro-symbolic AI Neuromorphic engineering Quantum machine learning Problems Classification Generative modeling Regression Clustering Dimensionality reduction Density estimation Anomaly detection Data cleaning AutoML Association rules Semantic analysis Structured prediction Feature engineering Feature learning Learning to rank Grammar induction Ontology learning Multimodal learning Supervised learning ( classification • regression ) Apprenticeship learning Decision trees Ensembles Bagging Boosting Random forest k -NN Linear regression Naive Bayes Artificial neural networks Logistic regression Perceptron Relevance vector machine (RVM) Support vector machine (SVM) Clustering BIRCH CURE Hierarchical k -means Fuzzy Expectation–maximization (EM) DBSCAN OPTICS Mean shift Dimensionality reduction Factor analysis CCA ICA LDA NMF PCA PGD t-SNE SDL Structured prediction Graphical models Bayes net Conditional random field Hidden Markov Anomaly detection RANSAC k -NN Local outlier factor Isolation forest Neural networks Autoencoder Deep learning Feedforward neural network Recurrent neural network LSTM GRU ESN reservoir computing Boltzmann machine Restricted GAN Diffusion model SOM Convolutional neural network U-Net LeNet AlexNet DeepDream Neural field Neural radiance field Physics-informed neural networks Transformer Vision Mamba Spiking neural network Memtransistor Electrochemical RAM (ECRAM) Reinforcement learning Q-learning Policy gradient SARSA Temporal difference (TD) Multi-agent Self-play Learning with humans Active learning Crowdsourcing Human-in-the-loop Mechanistic interpretability RLHF Model diagnostics Coefficient of determination Confusion matrix Learning curve ROC curve Mathematical foundations Kernel machines Bias–variance tradeoff Computational learning theory Empirical risk minimization Occam learning PAC learning Statistical learning VC theory Topological deep learning Journals and conferences AAAI CVPR ECCV ECML PKDD EMNLP ICCV NeurIPS ICML ICLR IJCAI ML JMLR Related articles Glossary of artificial intelligence List of datasets for machine-learning research List of datasets in computer vision and image processing Outline of machine learning v t e Adversarial machine learning is the study of the attacks on machine learning algorithms, and of the defenses against such attacks. [ 1 ] Machine learning techniques are mostly designed to work on specific problem sets, under the assumption that the training and test data are generated from the same statistical distribution ( IID ). However, this assumption is often violated in practical high-stake applications, where users may intentionally supply fabricated data that violates the statistical assumption. Most common attacks in adversarial machine learning include evasion attacks , [ 2 ] data poisoning attacks , [ 3 ] Byzantine attacks [ 4 ] and model extraction . [ 5 ] History [ edit ] At the MIT Spam Conference in January 2004, John Graham-Cumming showed that a machine-learning spam filter could be used to defeat another machine-learning spam filter by automatically learning which words to add to a spam email to get the email classified as not spam. [ 6 ] In 2004, Nilesh Dalvi and others noted that linear classifiers used in spam filters could be defeated by simple " evasion attacks" as spammers inserted "good words" into their spam emails. (Around 2007, some spammers added random noise to fuzz words within "image spam" in order to defeat OCR -based filters.) In 2006, Marco Barreno and others published "Can Machine Learning Be Secure?", outlining a broad taxonomy of attacks. As late as 2013 many researchers continued to hope that non-linear classifiers (such as support vector machines and neural networks ) might be robust to adversaries, until Battista Biggio and others demonstrated the first gradient-based attacks on such machine-learning models (2012 [ 7 ] –2013 [ 8 ] ). In 2012, deep neural networks began to dominate computer vision problems; starting in 2014, Christian Szegedy and others demonstrated that deep neural networks could be fooled by adversaries, again using a gradient-based attack to craft adversarial perturbations. [ 9 ] [ 10 ] Further work would show that adversarial attacks are harder to produce in uncontrolled environments, due to the different environmental constraints that cancel out the effect of noise. [ 11 ] [ 12 ] For example, any small rotation or slight illumination on an adversarial image can destroy the adversariality. In addition, researchers such as Google Brain 's Nick Frosst point out that it is much easier to make self-driving cars [ 13 ] miss stop signs by physically removing the sign itself, rather than creating adversarial examples. [ 14 ] Frosst also believes that the adversarial machine learning community incorrectly assumes models trained on a certain data distribution will also perform well on a completely different data distribution. He suggests that a new approach to machine learning should be explored, and is currently working on a unique neural network that has characteristics more similar to human perception than state-of-the-art approaches. [ 14 ] While adversarial machine learning continues to be heavily rooted in academia, large tech companies such as Google, Microsoft, and IBM have begun curating documentation and open source code bases to allow others to concretely assess the robustness of machine learning models and minimize the risk of adversarial attacks. [ 15 ] [ 16 ] [ 17 ] Examples [ edit ] Examples include attacks in spam filtering , where spam messages are obfuscated through the misspelling of "bad" words or the insertion of "good" words; [ 18 ] [ 19 ] attacks in computer security , such as obfuscating malware code within network packets or modifying the characteristics of a network flow to mislead intrusion detection; [ 20 ] [ 21 ] attacks in biometric recognition where fake biometric traits may be exploited to impersonate a legitimate user; [ 22 ] or to compromise users' template galleries that adapt to updated traits over time. Researchers showed that by changing only one pixel it was possible to fool deep learning algorithms. [ 23 ] Others 3-D printed a toy turtle with a texture engineered to make Google's object detection AI classify it as a rifle regardless of the angle from which the turtle was viewed. [ 24 ] Creating the turtle required only low-cost commercially available 3-D printing technology. [ 25 ] A machine-tweaked image of a dog was shown to look like a cat to both computers and humans. [ 26 ] A 2019 study reported that humans can guess how machines will classify adversarial images. [ 27 ] Researchers discovered methods for perturbing the appearance of a stop sign such that an autonomous vehicle classified it as a merge or speed limit sign. [ 13 ] [ 28 ] A data poisoning filter called Nightshade was released in 2023 by researchers at the University of Chicago . It was created for use by visual artists to put on their artwork to corrupt the data set of text-to-image models , which usually scrape their data from the internet without the consent of the image creator. [ 29 ] [ 30 ] McAfee attacked Tesla 's former Mobileye system, fooling it into driving 50 mph over the speed limit, simply by adding a two-inch strip of black tape to a speed limit sign. [ 31 ] [ 32 ] Adversarial patterns on glasses or clothing designed to deceive facial-recognition systems or license-plate readers, have led to a niche industry of "stealth streetwear". [ 33 ] An adversarial attack on a neural network can allow an attacker to inject algorithms into the target system. [ 34 ] Researchers can also create adversarial audio inputs to disguise commands to intelligent assistants in benign-seeming audio; [ 35 ] a parallel literature explores human perception of such stimuli. [ 36 ] [ 37 ] Clustering algorithms are used in security applications. Malware and computer virus analysis aims to identify malware families, and to generate specific detection signatures. [ 38 ] [ 39 ] In the context of malware detection, researchers have proposed methods for adversarial malware generation that automatically craft binaries to evade learning-based detectors while preserving malicious functionality. Optimization-based attacks such as GAMMA use genetic algorithms to inject benign content (for example, padding or new PE sections) into Windows executables, framing evasion as a constrained optimization problem that balances misclassification success with the size of the injected payload and showing transferability to commercial antivirus products. [ 40 ] Complementary work uses generative adversarial networks (GANs) to learn feature-space perturbations that cause malware to be classified as benign; Mal-LSGAN, for instance, replaces the standard GAN loss with a least-squares objective and modified activation functions to improve training stability and produce adversarial malware examples that substantially reduce true positive rates across multiple detectors. [ 41 ] Challenges in applying machine learning to security [ edit ] Researchers have observed that the constraints under which machine-learning techniques function in the security domain are different from those of common benchmark domains. Security data may change over time, include mislabeled samples, or reflect adversarial behavior, which complicates evaluation and reproducibility. [ 42 ] Data collection issues [ edit ] Security datasets vary across formats, including binaries, network traces, and log files. Studies have reported that the process of converting these sources into features can introduce bias or inconsistencies. [ 42 ] In addition, time-based leakage can occur when related malware samples are not properly separated across training and testing splits, which may lead to overly optimistic results. [ 42 ] Labeling and ground truth challenges [ edit ] Malware labels are often unstable because different antivirus engines may classify the same sample in conflicting ways. Ceschin et al. note that families may be renamed or reorganized over time, causing further discrepancies in ground truth and reducing the reliability of benchmarks. [ 42 ] Concept drift [ edit ] Because malware creators continuously adapt their techniques, the statistical properties of malicious samples also change. This form of concept drift has been widely documented and may reduce model performance unless systems are updated regularly or incorporate mechanisms for incremental learning . [ 42 ] Feature robustness [ edit ] Researchers differentiate between features that can be easily manipulated and those that are more resistant to modification. For example, simple static attributes, such as header fields, may be altered by attackers, while structural features, such as control-flow graphs , are generally more stable but computationally expensive to extract . [ 42 ] Class imbalance [ edit ] In realistic deployment environments, the proportion of malicious samples can be extremely low, ranging from 0.01% to 2% of total data. This unbalanced distribution causes models to develop a bias towards the majority class, achieving high accuracy but failing to identify malicious samples. [ 43 ] Prior approaches to this problem have included both data-level solutions and sequence-specific models. Methods like n-gram and Long Short-Term Memory (LSTM) networks can model sequential data, but their performance has been shown to decline significantly when malware samples are realistically proportioned in the training set, demonstrating the limitations in realistic security contexts. [ 43 ] To address this issue, one approach has been to adapt models from natural language processing , such as BERT . This method involves treating sequences of application activities as a form of "language" and fine-tuning a pre-trained BERT model on the specific task. A study applying this technique to Android activity sequences reported an F1 score of 0.919 on a dataset with only 0.5% malware samples. This result was a significant improvement over LSTM and n-gram models, demonstrating the potential of pre-trained models to handle class imbalance in malware detection. [ 43 ] System design and learning [ edit ] Feature engineering and training can causes issues. Data snooping is a common pitfall where a model is trained using information that would not be available in a real-world scenario. [ 44 ] Spurious correlations result when a model learns to associate artifacts with a label, rather than the underlying security-relevant pattern. [ 44 ] For example, a malware classifier might learn to identify a specific compiler artifact instead of malicious behavior itself. Biased parameter selection is a form of data snooping where model hyperparameters are tuned using the test set . [ 44 ] Performance evaluation [ edit ] The choice of the evaluation metrics can impact the validity of the results. Having an inappropriate baseline involves failing to compare a new model against simpler, well-established baselines. [ 44 ] Inappropriate performance measures means using metrics that do not align with the practical goals of the system. [ 44 ] Reporting only " accuracy " is often described as insufficient for an intrusion detection system, where false-positive rates are considered critically important. [ 44 ] Base rate fallacy is a failure to correctly interpret performance in the context of large class imbalances. [ 44 ] Deployment and operation [ edit ] Deployment introduces challenges related to the performance and security in live environments. Lab-only evaluation is the practice of evaluating a system only in a controlled, static laboratory setting, which does not account for real-world challenges like concept drift and performance overhead. [ 44 ] Inappropriate threat model refers to failing to consider the ML system itself as an attack surface . [ 44 ] Attack modalities [ edit ] Taxonomy [ edit ] Attacks against ( supervised ) machine learning algorithms have been categorized along three primary axes: [ 45 ] influence on the classifier, the security violation and their specificity. Classifier influence: An attack can influence the classifier by disrupting the classification phase. This may be preceded by an exploration phase to identify vulnerabilities. The attacker's capabilities might be restricted by the presence of data manipulation constraints. [ 46 ] Security violation: An attack can supply malicious data that gets classified as legitimate. Malicious data supplied during training can cause legitimate data to be rejected after training. Specificity: A targeted attack attempts to allow a specific intrusion/disruption. Alternatively, an indiscriminate attack creates general mayhem. This taxonomy has been extended into a more comprehensive threat model that allows explicit assumptions about the adversary's goal, knowledge of the attacked system, capability of manipulating the input data/system components, and on attack strategy. [ 47 ] [ 48 ] This taxonomy has further been extended to include dimensions for defense strategies against adversarial attacks. [ 49 ] Strategies [ edit ] Below are some of the most commonly encountered attack scenarios. Data poisoning [ edit ] Poisoning consists of contaminating the training dataset with data designed to increase errors in the output. Given that learning algorithms are shaped by their training datasets, poisoning can effectively reprogram algorithms with potentially malicious intent. Concerns have been raised especially for user-generated training data, e.g. for content recommendation or natural language models. The ubiquity of fake accounts offers many opportunities for poisoning. Facebook reportedly removes around 7 billion fake accounts per year. [ 50 ] [ 51 ] Poisoning has been reported as the leading concern for industrial applications. [ 52 ] On social medias, disinformation campaigns attempt to bias recommendation and moderation algorithms, to push certain content over others. [ further explanation needed ] A particular case of data poisoning is the backdoor attack, [ 53 ] which aims to teach a specific behavior for inputs with a given trigger, e.g. a small defect on images, sounds, videos or texts. AI training data poisoning illustration, showing how models can learn to associate given terms with wrong concepts due to data poisoning [ 54 ] For instance, intrusion detection systems are often trained using collected data. An attacker may poison this data by injecting malicious samples during operation that subsequently disrupt retraining. [ 47 ] [ 48 ] [ 45 ] [ 55 ] [ 56 ] Data poisoning techniques can also be applied to text-to-image models to alter their output, which is used by artists to defend their copyrighted works or their artistic style against imitation. [ 29 ] Data poisoning can also happen unintentionally through model collapse , where models are trained on synthetic data. [ 57 ] Byzantine attacks [ edit ] As machine learning is scaled, it often relies on multiple computing machines. In federated learning , for instance, edge devices collaborate with a central server, typically by sending gradients or model parameters. However, some of these devices may deviate from their expected behavior, e.g. to harm the central server's model [ 58 ] or to bias algorithms towards certain behaviors (e.g., amplifying the recommendation of disinformation content). On the other hand, if the training is performed on a single machine, then the model is very vulnerable to a failure of the machine, or an attack on the machine; the machine is a single point of failure . [ 59 ] In fact, the machine owner may themselves insert provably undetectable backdoors . [ 60 ] The current leading solutions to make (distributed) learning algorithms provably resilient to a minority of malicious (a.k.a. Byzantine ) participants are based on robust gradient aggregation rules. [ 61 ] [ 62 ] [ 63 ] [ 64 ] [ 65 ] [ 66 ] The robust aggregation rules do not always work especially when the data across participants has a non-iid distribution. Nevertheless, in the context of heterogeneous honest participants, such as users with different consumption habits for recommendation algorithms or writing styles for language models, there are provable impossibility theorems on what any robust learning algorithm can guarantee. [ 4 ] [ 67 ] Evasion [ edit ] Evasion attacks [ 8 ] [ 47 ] [ 48 ] [ 68 ] consist of exploiting the imperfection of a trained model. For instance, spammers and hackers often attempt to evade detection by obfuscating the content of spam emails and malware . Samples are modified to evade detection; that is, to be classified as legitimate. This does not involve influence over the training data. A clear example of evasion is image-based spam in which the spam content is embedded within an attached image to evade textual analysis by anti-spam filters. Another example of evasion is given by spoofing attacks against biometric verification systems. [ 22 ] Evasion attacks can be generally split into two different categories: black box attacks and white box attacks . [ 16 ] Model extraction [ edit ] Model extraction involves an adversary probing a black box machine learning system in order to extract the data it was trained on. [ 69 ] [ 70 ] This can cause issues when either the training data or the model itself is sensitive and confidential. For example, model extraction could be used to extract a proprietary stock trading model which the adversary could then use for their own financial benefit. In the extreme case, model extraction can lead to model stealing, which corresponds to extracting a sufficient amount of data from the model to enable the complete reconstruction of the model. On the other hand, membership inference is a targeted model extraction attack, which infers the owner of a data point, often by leveraging the overfitting resulting from poor machine learning practices. [ 71 ] Concerningly, this is sometimes achievable even without knowledge or access to a target model's parameters, raising security concerns for models trained on sensitive data, including but not limited to medical records and/or personally identifiable information. With the emergence of transfer learning and public accessibility of many state of the art machine learning models, tech companies are increasingly drawn to create models based on public ones, giving attackers freely accessible information to the structure and type of model being used. [ 71 ] Categories [ edit ] Adversarial attacks and training in linear models [ edit ] There is a growing literature about adversarial attacks in linear models . Indeed, since the seminal work from Goodfellow at al. [ 72 ] studying these models in linear models has been an important tool to understand how adversarial attacks affect machine learning models. The analysis of these models is simplified because the computation of adversarial attacks can be simplified in linear regression and classification problems. Moreover, adversarial training is convex in this case. [ 73 ] Linear models allow for analytical analysis while still reproducing phenomena observed in state-of-the-art models. One prime example of that is how this model can be used to explain the trade-off between robustness and accuracy . [ 74 ] Diverse work indeed provides analysis of adversarial attacks in linear models, including asymptotic analysis for classification [ 75 ] and for linear regression. [ 76 ] [ 77 ] And, finite-sample analysis based on Rademacher complexity . [ 78 ] A result from studying adversarial attacks in linear models is that it closely relates to regularization . [ 79 ] Under certain conditions, it has been shown that adversarial training of a linear regression model with input perturbations restricted by the infinity-norm closely resembles Lasso regression, and that adversarial training of a linear regression model with input perturbations restricted by the 2-norm closely resembles Ridge regression . Adversarial deep reinforcement learning [ edit ] Adversarial deep reinforcement learning is an active area of research in reinforcement learning focusing on vulnerabilities of learned policies. In this research area, some studies initially showed that reinforcement learning policies are susceptible to imperceptible adversarial manipulations. [ 72 ] [ 80 ] While some methods have been proposed to overcome these susceptibilities, in the most recent studies it has been shown that these proposed solutions are far from providing an accurate representation of current vulnerabilities of deep reinforcement learning policies. [ 81 ] Adversarial natural language processing [ edit ] Adversarial attacks on speech recognition have been introduced for speech-to-text applications, in particular for Mozilla's implementation of DeepSpeech [ 82 ] and Google's Speech Commands model. [ 83 ] Various pre-processing techniques have been proposed to defend against such attacks [ 84 ] , but these methods may not be resilient to attackers aware of those defenses. Specific attack types [ edit ] There are a large variety of different adversarial attacks that can be used against machine learning systems. Many of these work on both deep learning systems as well as traditional machine learning models such as SVMs [ 7 ] and linear regression . [ 85 ] A high level sample of these attack types include: Adversarial examples [ 86 ] Trojan and backdoor attacks [ 87 ] Model inversion [ 88 ] Membership inference [ 89 ] Adversarial examples [ edit ] An adversarial example refers to specially crafted input that is designed to look "normal" to humans but causes misclassification to a machine learning model. Often, a form of specially designed "noise" is used to elicit the misclassifications. Below are some current techniques for generating adversarial examples in the literature (by no means an exhaustive list). Gradient-based evasion attack [ 8 ] Fast Gradient Sign Method (FGSM) [ 90 ] Projected Gradient Descent (PGD) [ 91 ] Carlini and Wagner (C&W) attack [ 92 ] Adversarial patch attack [ 93 ] Black box attacks [ edit ] Black box attacks in adversarial machine learning assume that the adversary can only get outputs for provided inputs and has no knowledge of the model structure or parameters. [ 16 ] [ 94 ] In this case, the adversarial example is generated either using a model created from scratch, or without any model at all (excluding the ability to query the original model). In either case, the objective of these attacks is to create adversarial examples that are able to transfer to the black box model in question. [ 95 ] Simple black-box adversarial attacks [ edit ] Simple black-box adversarial attacks is a query-efficient way to attack black-box image classifiers. [ 96 ] Take a random orthonormal basis v 1 , v 2 , … , v d {\displaystyle v_{1},v_{2},\dots ,v_{d}} in R d {\displaystyle \mathbb {R} ^{d}} . The authors suggested the discrete cosine transform of the standard basis (the pixels). For a correctly classified image x {\displaystyle x} , try x + ϵ v 1 , x − ϵ v 1 {\displaystyle x+\epsilon v_{1},x-\epsilon v_{1}} , and compare the amount of error in the classifier upon x + ϵ v 1 , x , x − ϵ v 1 {\displaystyle x+\epsilon v_{1},x,x-\epsilon v_{1}} . Pick the one that causes the largest amount of error. Repeat this for v 2 , v 3 , … {\displaystyle v_{2},v_{3},\dots } until the desired level of error in the classifier is reached. It was discovered when the authors designed a simple baseline to compare with a previous black-box adversarial attack algorithm based on gaussian processes , and were surprised that the baseline worked even better. [ 97 ] Square attack [ edit ] Square attacks were introduced in 2020 as a black-box evasion adversarial attack technique based on querying classification scores without the need for gradient information. [ 98 ] As a score based black box attack, this adversarial approach is able to query probability distributions across model output classes, but has no other access to the model itself. According to the paper's authors, the proposed Square Attack required fewer queries than when compared to state-of-the-art score-based black box attacks at the time. [ 98 ] To describe the function objective, the attack defines the classifier as f : [ 0 , 1 ] d → R K {\textstyle f:[0,1]^{d}\rightarrow \mathbb {R} ^{K}} , with d {\textstyle d} representing the dimensions of the input and K {\textstyle K} as the total number of output classes. f k ( x ) {\textstyle f_{k}(x)} returns the score (or a probability between 0 and 1) that the input x {\textstyle x} belongs to class k {\textstyle k} , which allows the classifier's class output for any input x {\textstyle x} to be defined as argmax k = 1 , . . . , K f k ( x ) {\textstyle {\text{argmax}}_{k=1,...,K}f_{k}(x)} . The goal of this attack is as follows: [ 98 ] argmax k = 1 , . . . , K f k ( x ^ ) ≠ y , | | x ^ − x | | p ≤ ϵ and x ^ ∈ [ 0 , 1 ] d {\displaystyle {\text{argmax}}_{k=1,...,K}f_{k}({\hat {x}})\neq y,||{\hat {x}}-x||_{p}\leq \epsilon {\text{ and }}{\hat {x}}\in [0,1]^{d}} In other words, finding some perturbed adversarial example x ^ {\textstyle {\hat {x}}} such that the classifier incorrectly classifies it to some other class under the constraint that x ^ {\textstyle {\hat {x}}} and x {\textstyle x} are similar. The paper then defines loss L {\textstyle L} as L ( f ( x ^ ) , y ) = f y ( x ^ ) − max k ≠ y f k ( x ^ ) {\textstyle L(f({\hat {x}}),y)=f_{y}({\hat {x}})-\max _{k\neq y}f_{k}({\hat {x}})} and proposes the solution to finding adversarial example x ^ {\textstyle {\hat {x}}} as solving the below constrained optimization problem : [ 98 ] min x ^ ∈ [ 0 , 1 ] d L ( f ( x ^ ) , y ) , s.t. | | x ^ − x | | p ≤ ϵ {\displaystyle \min _{{\hat {x}}\in [0,1]^{d}}L(f({\hat {x}}),y),{\text{ s.t. }}||{\hat {x}}-x||_{p}\leq \epsilon } The result in theory is an adversarial example that is highly confident in the incorrect class but is also very similar to the original image. To find such example, Square Attack utilizes the iterative random search technique to randomly perturb the image in hopes of improving the objective function. In each step, the algorithm perturbs only a small square section of pixels, hence the name Square Attack, which terminates as soon as an adversarial example is found in order to improve query efficiency. Finally, since the attack algorithm uses scores and not gradient information, the authors of the paper indicate that this approach is not affected by gradient masking, a common technique formerly used to prevent evasion attacks. [ 98 ] HopSkipJump attack [ edit ] This black box attack was also proposed as a query efficient attack, but one that relies solely on access to any input's predicted output class. In other words, the HopSkipJump attack does not require the ability to calculate gradients or access to score values like the Square Attack, and will require just the model's class prediction output (for any given input). The proposed attack is split into two different settings, targeted and untargeted, but both are built from the general idea of adding minimal perturbations that leads to a different model output. In the targeted setting, the goal is to cause the model to misclassify the perturbed image to a specific target label (that is not the original label). In the untargeted setting, the goal is to cause the model to misclassify the perturbed image to any label that is not the original label. The attack objectives for both are as follows where x {\textstyle x} is the original image, x ′ {\textstyle x^{\prime }} is the adversarial image, d {\textstyle d} is a distance function between images, c ∗ {\textstyle c^{*}} is the target label, and C {\textstyle C} is the model's classification class label function: [ 99 ] Targeted: min x ′ d ( x ′ , x ) subject to C ( x ′ ) = c ∗ {\displaystyle {\textbf {Targeted:}}\min _{x^{\prime }}d(x^{\prime },x){\text{ subject to }}C(x^{\prime })=c^{*}} Untargeted: min x ′ d ( x ′ , x ) subject to C ( x ′ ) ≠ C ( x ) {\displaystyle {\textbf {Untargeted:}}\min _{x^{\prime }}d(x^{\prime },x){\text{ subject to }}C(x^{\prime })\neq C(x)} To solve this problem, the attack proposes the following boundary function S {\textstyle S} for both the untargeted and targeted setting: [ 99 ] S ( x ′ ) := { max c ≠ C ( x ) F ( x ′ ) c − F ( x ′ ) C ( x ) , (Untargeted) F ( x ′ ) c ∗ − max c ≠ c ∗ F ( x ′ ) c , (Targeted) {\displaystyle S(x^{\prime }):={\begin{cases}\max _{c\neq C(x)}{F(x^{\prime })_{c}}-F(x^{\prime })_{C(x)},&{\text{(Untargeted)}}\\F(x^{\prime })_{c^{*}}-\max _{c\neq c^{*}}{F(x^{\prime })_{c}},&{\text{(Targeted)}}\end{cases}}} This can be further simplified to better visualize the boundary between different potential adversarial examples: [ 99 ] 0 \\iff \\begin{cases} argmax_cF(x^\\prime) \\neq C(x) , & \\text{(Untargeted)} \\\\ argmax_cF(x^\\prime) = c^*, & \\text{(Targeted)} \\end{cases} "}}'> 0\iff {\begin{cases}argmax_{c}F(x^{\prime })\neq C(x),&{\text{(Untargeted)}}\\argmax_{c}F(x^{\prime })=c^{*},&{\text{(Targeted)}}\end{cases}}}"> S ( x ′ ) > 0 ⟺ { a r g m a x c F ( x ′ ) ≠ C ( x ) , (Untargeted) a r g m a x c F ( x ′ ) = c ∗ , (Targeted) {\displaystyle S(x^{\prime })>0\iff {\begin{cases}argmax_{c}F(x^{\prime })\neq C(x),&{\text{(Untargeted)}}\\argmax_{c}F(x^{\prime })=c^{*},&{\text{(Targeted)}}\end{cases}}} 0\iff {\begin{cases}argmax_{c}F(x^{\prime })\neq C(x),&{\text{(Untargeted)}}\\argmax_{c}F(x^{\prime })=c^{*},&{\text{(Targeted)}}\end{cases}}}"/> With this boundary function, the attack then follows an iterative algorithm to find adversarial examples x ′ {\textstyle x^{\prime }} for a given image x {\textstyle x} that satisfies the attack objectives. Initialize x {\textstyle x} to some point where 0"}}'> 0}"> S ( x ) > 0 {\textstyle S(x)>0} 0}"/> Iterate below Boundary search Gradient update Compute the gradient Find the step size Boundary search uses a modified binary search to find the point in which the boundary (as defined by S {\textstyle S} ) intersects with the line between x {\textstyle x} and x ′ {\textstyle x^{\prime }} . The next step involves calculating the gradient for x {\textstyle x} , and update the original x {\textstyle x} using this gradient and a pre-chosen step size. HopSkipJump authors prove that this iterative algorithm will converge, leading x {\textstyle x} to a point right along the boundary that is very close in distance to the original image. [ 99 ] However, since HopSkipJump is a proposed black box attack and the iterative algorithm above requires the calculation of a gradient in the second iterative step (which black box attacks do not have access to), the authors propose a solution to gradient calculation that requires only the model's output predictions alone. [ 99 ] By generating many random vectors in all directions, denoted as u b {\textstyle u_{b}} , an approximation of the gradient can be calculated using the average of these random vectors weighted by the sign of the boundary function on the image x ′ + δ u b {\textstyle x^{\prime }+\delta _{u_{b}}} , where δ u b {\textstyle \delta _{u_{b}}} is the size of the random vector perturbation: [ 99 ] ∇ S ( x ′ , δ ) ≈ 1 B ∑ b = 1 B ϕ ( x ′ + δ u b ) u b {\displaystyle \nabla S(x^{\prime },\delta )\approx {\frac {1}{B}}\sum _{b=1}^{B}\phi (x^{\prime }+\delta _{u_{b}})u_{b}} The result of the equation above gives a close approximation of the gradient required in step 2 of the iterative algorithm, completing HopSkipJump as a black box attack. [ 100 ] [ 101 ] [ 99 ] White box attacks [ edit ] White box attacks assumes that the adversary has access to model parameters on top of being able to get labels for provided inputs. [ 95 ] Fast gradient sign method [ edit ] One of the first proposed attacks for generating adversarial examples was proposed by Google researchers Ian J. Goodfellow , Jonathon Shlens, and Christian Szegedy. [ 102 ] The attack was called fast gradient sign method (FGSM), and it consists of adding a linear amount of in-perceivable noise to the image and causing a model to incorrectly classify it. This noise is calculated by multiplying the sign of the gradient with respect to the image we want to perturb by a small constant epsilon. As epsilon increases, the model is more likely to be fooled, but the perturbations become easier to identify as well. Shown below is the equation to generate an adversarial example where x {\textstyle x} is the original image, ϵ {\textstyle \epsilon } is a very small number, Δ x {\textstyle \Delta _{x}} is the gradient function, J {\textstyle J} is the loss function, θ {\textstyle \theta } is the model weights, and y {\textstyle y} is the true label. [ 103 ] a d v x = x + ϵ ⋅ s i g n ( Δ x J ( θ , x , y ) ) {\displaystyle adv_{x}=x+\epsilon \cdot sign(\Delta _{x}J(\theta ,x,y))} One important property of this equation is that the gradient is calculated with respect to the input image since the goal is to generate an image that maximizes the loss for the original image of true label y {\textstyle y} . In traditional gradient descent (for model training), the gradient is used to update the weights of the model since the goal is to minimize the loss for the model on a ground truth dataset. The Fast Gradient Sign Method was proposed as a fast way to generate adversarial examples to evade the model, based on the hypothesis that neural networks cannot resist even linear amounts of perturbation to the input. [ 104 ] [ 103 ] [ 102 ] FGSM has shown to be effective in adversarial attacks for image classification and skeletal action recognition. [ 105 ] Carlini & Wagner [ edit ] In an effort to analyze existing adversarial attacks and defenses, researchers at the University of California, Berkeley, Nicholas Carlini and David Wagner in 2016 propose a faster and more robust method to generate adversarial examples. [ 106 ] The attack proposed by Carlini and Wagner begins with trying to solve a difficult non-linear optimization equation: [ 70 ] min ( | | δ | | p ) subject to C ( x + δ ) = t , x + δ ∈ [ 0 , 1 ] n {\displaystyle \min(||\delta ||_{p}){\text{ subject to }}C(x+\delta )=t,x+\delta \in [0,1]^{n}} Here the objective is to minimize the noise ( δ {\textstyle \delta } ), added to the original input x {\textstyle x} , such that the machine learning algorithm ( C {\textstyle C} ) predicts the original input with delta (or x + δ {\textstyle x+\delta } ) as some other class t {\textstyle t} . However instead of directly the above equation, Carlini and Wagner propose using a new function f {\textstyle f} such that: [ 70 ] C ( x + δ ) = t ⟺ f ( x + δ ) ≤ 0 {\displaystyle C(x+\delta )=t\iff f(x+\delta )\leq 0} This condenses the first equation to the problem below: [ 70 ] min ( | | δ | | p ) subject to f ( x + δ ) ≤ 0 , x + δ ∈ [ 0 , 1 ] n {\displaystyle \min(||\delta ||_{p}){\text{ subject to }}f(x+\delta )\leq 0,x+\delta \in [0,1]^{n}} and even more to the equation below: [ 70 ] min ( | | δ | | p + c ⋅ f ( x + δ ) ) , x + δ ∈ [ 0 , 1 ] n {\displaystyle \min(||\delta ||_{p}+c\cdot f(x+\delta )),x+\delta \in [0,1]^{n}} Carlini and Wagner then propose the use of the below function in place of f {\textstyle f} using Z {\textstyle Z} , a function that determines class probabilities for given input x {\textstyle x} . When substituted in, this equation can be thought of as finding a target class that is more confident than the next likeliest class by some constant amount: [ 70 ] f ( x ) = ( [ max i ≠ t Z ( x ) i ] − Z ( x ) t ) + {\displaystyle f(x)=([\max _{i\neq t}Z(x)_{i}]-Z(x)_{t})^{+}} When solved using gradient descent, this equation is able to produce stronger adversarial examples when compared to fast gradient sign method that is also able to bypass defensive distillation , a defense that was once proposed to be effective against adversarial examples. [ 107 ] [ 108 ] [ 106 ] [ 70 ] Defenses [ edit ] Conceptual representation of the proactive arms race [ 48 ] [ 39 ] Researchers have proposed a multi-step approach to protecting machine learning. [ 10 ] Threat modeling – formalize the attackers goals and capabilities with respect to the target system. Attack simulation – formalize the optimization problem the attacker tries to solve according to possible attack strategies. Attack impact evaluation Countermeasure design Noise detection (for evasion-based attack) [ 109 ] Information laundering – Alter the information received by adversaries (for model stealing attacks) [ 70 ] Mechanisms [ edit ] A number of defense mechanisms against evasion, poisoning, and privacy attacks have been proposed, including: Secure learning algorithms [ 19 ] [ 110 ] [ 111 ] Byzantine-resilient algorithms [ 61 ] [ 4 ] Multiple classifier systems [ 18 ] [ 112 ] AI-written algorithms. [ 34 ] AIs that explore the training environment; for example, in image recognition, actively navigating a 3D environment rather than passively scanning a fixed set of 2D images. [ 34 ] Privacy-preserving learning [ 48 ] [ 113 ] Ladder algorithm for Kaggle -style competitions [ 114 ] Game theoretic models [ 115 ] [ 116 ] [ 117 ] Sanitizing training data Adversarial training [ 90 ] [ 21 ] Backdoor detection algorithms [ 118 ] Gradient masking/obfuscation techniques: to prevent the adversary exploiting the gradient in white-box attacks. This family of defenses is deemed unreliable as these models are still vulnerable to black-box attacks or can be circumvented in other ways. [ 119 ] Ensembles of models have been proposed in literature, which have shown to be ineffective against evasion attacks [ 120 ] but effective against data poisoning attacks. [ 121 ] Moving target defense [ 122 ] See also [ edit ] Pattern recognition Fawkes (image cloaking software) References [ edit ] ↑ Kianpour, Mazaher; Wen, Shao-Fang (2020). "Timing Attacks on Machine Learning: State of the Art". Intelligent Systems and Applications . Advances in Intelligent Systems and Computing. Vol. 1037. pp. 111– 125. doi : 10.1007/978-3-030-29516-5_10 . ISBN 978-3-030-29515-8 . S2CID 201705926 . ↑ Goodfellow, Ian; McDaniel, Patrick; Papernot, Nicolas (25 June 2018). "Making machine learning robust against adversarial inputs" . Communications of the ACM . 61 (7): 56– 66. doi : 10.1145/3134599 . ISSN 0001-0782 . [ permanent dead link ] ↑ Geiping, Jonas; Fowl, Liam H.; Huang, W. Ronny; Czaja, Wojciech; Taylor, Gavin; Moeller, Michael; Goldstein, Tom (2020-09-28). Witches' Brew: Industrial Scale Data Poisoning via Gradient Matching . International Conference on Learning Representations 2021 (Poster). 1 2 3 El-Mhamdi, El Mahdi; Farhadkhani, Sadegh; Guerraoui, Rachid; Guirguis, Arsany; Hoang, Lê-Nguyên; Rouault, Sébastien (2021-12-06). "Collaborative Learning in the Jungle (Decentralized, Byzantine, Heterogeneous, Asynchronous and Nonconvex Learning)" . Advances in Neural Information Processing Systems . 34 . arXiv : 2008.00742 . ↑ Tramèr, Florian; Zhang, Fan; Juels, Ari; Reiter, Michael K.; Ristenpart, Thomas (2016). Stealing Machine Learning Models via Prediction {APIs} . 25th USENIX Security Symposium. pp. 601– 618. ISBN 978-1-931971-32-4 . ↑ "How to beat an adaptive/Bayesian spam filter (2004)" . Retrieved 2023-07-05 . 1 2 Biggio, Battista; Nelson, Blaine; Laskov, Pavel (2013-03-25). "Poisoning Attacks against Support Vector Machines". arXiv : 1206.6389 [ cs.LG ]. 1 2 3 Biggio, Battista; Corona, Igino; Maiorca, Davide; Nelson, Blaine; Srndic, Nedim; Laskov, Pavel; Giacinto, Giorgio; Roli, Fabio (2013). "Evasion Attacks against Machine Learning at Test Time". Machine Learning and Knowledge Discovery in Databases . Lecture Notes in Computer Science. Vol. 8190. Springer. pp. 387– 402. arXiv : 1708.06131 . doi : 10.1007/978-3-642-40994-3_25 . ISBN 978-3-642-38708-1 . S2CID 18716873 . ↑ Szegedy, Christian; Zaremba, Wojciech; Sutskever, Ilya; Bruna, Joan; Erhan, Dumitru; Goodfellow, Ian; Fergus, Rob (2014-02-19). "Intriguing properties of neural networks". arXiv : 1312.6199 [ cs.CV ]. 1 2 Biggio, Battista; Roli, Fabio (December 2018). "Wild patterns: Ten years after the rise of adversarial machine learning". Pattern Recognition . 84 : 317– 331. arXiv : 1712.03141 . Bibcode : 2018PatRe..84..317B . doi : 10.1016/j.patcog.2018.07.023 . S2CID 207324435 . ↑ Kurakin, Alexey; Goodfellow, Ian; Bengio, Samy (2016). "Adversarial examples in the physical world". arXiv : 1607.02533 [ cs.CV ]. ↑ Gupta, Kishor Datta; Dasgupta, Dipankar; Akhtar, Zahid (2020). Applicability issues of Evasion-Based Adversarial Attacks and Mitigation Techniques . 2020 IEEE Symposium Series on Computational Intelligence (SSCI). doi : 10.1109/SSCI47803.2020.9308589 . 1 2 Lim, Hazel Si Min; Taeihagh, Araz (2019). "Algorithmic Decision-Making in AVs: Understanding Ethical and Technical Concerns for Smart Cities" . Sustainability . 11 (20): 5791. arXiv : 1910.13122 . Bibcode : 2019arXiv191013122L . doi : 10.3390/su11205791 . S2CID 204951009 . 1 2 "Google Brain's Nicholas Frosst on Adversarial Examples and Emotional Responses" . Synced . 2019-11-21 . Retrieved 2021-10-23 . ↑ "Responsible AI practices" . Google AI . Retrieved 2021-10-23 . 1 2 3 Adversarial Robustness Toolbox (ART) v1.8 , Trusted-AI, 2021-10-23 , retrieved 2021-10-23 ↑ amarshal. "Failure Modes in Machine Learning - Security documentation" . docs.microsoft.com . Retrieved 2021-10-23 . 1 2 Biggio, Battista; Fumera, Giorgio; Roli, Fabio (2010). "Multiple classifier systems for robust classifier design in adversarial environments" . International Journal of Machine Learning and Cybernetics . 1 ( 1– 4): 27– 41. doi : 10.1007/s13042-010-0007-7 . hdl : 11567/1087824 . ISSN 1868-8071 . S2CID 8729381 . Archived from the original on 2023-01-19 . Retrieved 2015-01-14 . 1 2 Brückner, Michael; Kanzow, Christian; Scheffer, Tobias (2012). "Static Prediction Games for Adversarial Learning Problems" (PDF) . Journal of Machine Learning Research . 13 (Sep): 2617– 2654. ISSN 1533-7928 . ↑ Apruzzese, Giovanni; Andreolini, Mauro; Ferretti, Luca; Marchetti, Mirco; Colajanni, Michele (2021-06-03). "Modeling Realistic Adversarial Attacks against Network Intrusion Detection Systems". Digital Threats: Research and Practice . 3 (3): 1– 19. arXiv : 2106.09380 . doi : 10.1145/3469659 . ISSN 2692-1626 . S2CID 235458519 . 1 2 Vitorino, João; Oliveira, Nuno; Praça, Isabel (March 2022). "Adaptative Perturbation Patterns: Realistic Adversarial Learning for Robust Intrusion Detection" . Future Internet . 14 (4): 108. arXiv : 2203.04234 . doi : 10.3390/fi14040108 . hdl : 10400.22/21851 . ISSN 1999-5903 . 1 2 Rodrigues, Ricardo N.; Ling, Lee Luan; Govindaraju, Venu (1 June 2009). "Robustness of multimodal biometric fusion methods against spoof attacks" (PDF) . Journal of Visual Languages & Computing . 20 (3): 169– 179. doi : 10.1016/j.jvlc.2009.01.010 . ISSN 1045-926X . ↑ Su, Jiawei; Vargas, Danilo Vasconcellos; Sakurai, Kouichi (October 2019). "One Pixel Attack for Fooling Deep Neural Networks". IEEE Transactions on Evolutionary Computation . 23 (5): 828– 841. arXiv : 1710.08864 . Bibcode : 2019ITEC...23..828S . doi : 10.1109/TEVC.2019.2890858 . ISSN 1941-0026 . S2CID 2698863 . ↑ "Single pixel change fools AI programs" . BBC News . 3 November 2017 . Retrieved 12 February 2018 . ↑ Athalye, Anish; Engstrom, Logan; Ilyas, Andrew; Kwok, Kevin (2017). "Synthesizing Robust Adversarial Examples". arXiv : 1707.07397 [ cs.CV ]. ↑ "AI Has a Hallucination Problem That's Proving Tough to Fix" . WIRED . 2018 . Retrieved 10 March 2018 . ↑ Zhou, Zhenglong; Firestone, Chaz (2019). "Humans can decipher adversarial images" . Nature Communications . 10 (1): 1334. arXiv : 1809.04120 . Bibcode : 2019NatCo..10.1334Z . doi : 10.1038/s41467-019-08931-6 . PMC 6430776 . PMID 30902973 . ↑ Ackerman, Evan (2017-08-04). "Slight Street Sign Modifications Can Completely Fool Machine Learning Algorithms" . IEEE Spectrum: Technology, Engineering, and Science News . Retrieved 2019-07-15 . 1 2 Edwards, Benj (2023-10-25). "University of Chicago researchers seek to "poison" AI art generators with Nightshade" . Ars Technica . Retrieved 2023-10-27 . ↑ Shan, Shawn; Ding, Wenxin; Passananti, Josephine; Wu, Stanley; Zheng, Haitao; Zhao, Ben Y. (2023). "Nightshade: Prompt-Specific Poisoning Attacks on Text-to-Image Generative Models". arXiv : 2310.13828 [ cs.CR ]. ↑ "A Tiny Piece of Tape Tricked Teslas Into Speeding Up 50 MPH" . Wired . 2020 . Retrieved 11 March 2020 . ↑ "Model Hacking ADAS to Pave Safer Roads for Autonomous Vehicles" . McAfee Blogs . 2020-02-19 . Retrieved 2026-01-04 . ↑ Seabrook, John (2020). "Dressing for the Surveillance Age" . The New Yorker . Retrieved 5 April 2020 . 1 2 3 Heaven, Douglas (October 2019). "Why deep-learning AIs are so easy to fool". Nature . 574 (7777): 163– 166. Bibcode : 2019Natur.574..163H . doi : 10.1038/d41586-019-03013-5 . PMID 31597977 . S2CID 203928744 . ↑ Hutson, Matthew (10 May 2019). "AI can now defend itself against malicious messages hidden in speech". Nature . doi : 10.1038/d41586-019-01510-1 . PMID 32385365 . S2CID 189666088 . ↑ Lepori, Michael A; Firestone, Chaz (2020-03-27). "Can you hear me now? Sensitive comparisons of human and machine perception". arXiv : 2003.12362 [ eess.AS ]. ↑ Vadillo, Jon; Santana, Roberto (2020-01-23). "On the human evaluation of audio adversarial examples". arXiv : 2001.08444 [ eess.AS ]. ↑ Skillicorn, D. B. (2009). "Adversarial knowledge discovery". IEEE Intelligent Systems . 24 : 54– 61. doi : 10.1109/MIS.2009.82 . 1 2 Biggio, B.; Fumera, G.; Roli., F. (2014). "Pattern recognition systems under attack: Design issues and research challenges" . Int'l J. Patt. Recogn. Artif. Intell . 28 (7): 1460002. doi : 10.1142/S0218001414600027 . Archived from the original on 2022-05-20. ↑ Demetrio, L.; Biggio, B.; Lagorio, G.; Roli, F.; Armando, A. (2021). "Functionality-Preserving Black-Box Optimization of Adversarial Windows Malware". IEEE Transactions on Information Forensics and Security . 16 : 3469– 3478. arXiv : 2003.13526 . Bibcode : 2021ITIF...16.3469D . doi : 10.1109/TIFS.2021.3082330 . ↑ Wang, J.; Chang, X.; Mišić, J.; Mišić, V. B.; Wang, Y.; Zhang, J. (2021). Mal-LSGAN: An Effective Adversarial Malware Example Generation Model . IEEE GLOBECOM. doi : 10.1109/GLOBECOM46510.2021.9685442 . 1 2 3 4 5 6 Ceschin, Fabrício; Botacin, Marcus; Bifet, Albert; Pfahringer, Bernhard; Oliveira, Luiz S.; Gomes, Heitor Murilo; Grégio, André (2023). "Machine Learning (In) Security: A Stream of Problems". Digital Threats: Research and Practice . 1 (1): 1– 32. arXiv : 2010.16045 . doi : 10.1145/3617897 . 1 2 3 Oak, Rajvardhan; Du, Min; Yan, David; Takawale, Harshvardhan; Amit, Idan (2019-11-11). "Malware Detection on Highly Imbalanced Data through Sequence Modeling" . Proceedings of the 12th ACM Workshop on Artificial Intelligence and Security . ACM. pp. 37– 48. doi : 10.1145/3338501.3357374 . ISBN 978-1-4503-6833-9 . 1 2 3 4 5 6 7 8 9 Arp, Daniel; Quiring, Erwin; Pendlebury, Feargus; Warnecke, Alexander; Pierazzi, Fabio; Wressnegger, Christian; Cavallaro, Lorenzo; Rieck, Konrad (2021-11-30), Dos and Don'ts of Machine Learning in Computer Security , arXiv : 2010.09470 1 2 Barreno, Marco; Nelson, Blaine; Joseph, Anthony D.; Tygar, J. D. (2010). "The security of machine learning" (PDF) . Machine Learning . 81 (2): 121– 148. Bibcode : 2010MLear..81..121B . doi : 10.1007/s10994-010-5188-5 . S2CID 2304759 . ↑ Sikos, Leslie F. (2019). AI in Cybersecurity . Intelligent Systems Reference Library. Vol. 151. Cham: Springer. p. 50. doi : 10.1007/978-3-319-98842-9 . ISBN 978-3-319-98841-2 . S2CID 259216663 . 1 2 3 Biggio, B.; Fumera, G.; Roli, F. (2014). "Security evaluation of pattern classifiers under attack" . IEEE Transactions on Knowledge and Data Engineering . 26 (4): 984– 996. arXiv : 1709.00609 . Bibcode : 2014ITKDE..26..984B . doi : 10.1109/TKDE.2013.57 . Archived from the original on 2018-05-18. 1 2 3 4 5 Biggio, Battista; Corona, Igino; Nelson, Blaine; Rubinstein, Benjamin I. P.; Maiorca, Davide; Fumera, Giorgio; Giacinto, Giorgio; Roli, Fabio (2014). "Security Evaluation of Support Vector Machines in Adversarial Environments". Support Vector Machines Applications . Springer International Publishing. pp. 105– 153. arXiv : 1401.7727 . doi : 10.1007/978-3-319-02300-7_4 . ISBN 978-3-319-02300-7 . S2CID 18666561 . ↑ Heinrich, Kai; Graf, Johannes; Chen, Ji; Laurisch, Jakob; Zschech, Patrick (2020-06-15). "Fool Me Once, Shame On You, Fool Me Twice, Shame On Me: A Taxonomy of Attack and De-fense Patterns for AI Security" . ECIS 2020 Research Papers . ↑ "Facebook removes 15 Billion fake accounts in two years" . Tech Digest . 2021-09-27 . Retrieved 2022-06-08 . ↑ "Facebook removed 3 billion fake accounts in just 6 months" . New York Post . Associated Press. 2019-05-23 . Retrieved 2022-06-08 . ↑ Siva Kumar, Ram Shankar; Nyström, Magnus; Lambert, John; Marshall, Andrew; Goertzel, Mario; Comissoneru, Andi; Swann, Matt; Xia, Sharon (May 2020). "Adversarial Machine Learning-Industry Perspectives". 2020 IEEE Security and Privacy Workshops (SPW) . pp. 69– 75. doi : 10.1109/SPW50608.2020.00028 . ISBN 978-1-7281-9346-5 . S2CID 229357721 . ↑ Schwarzschild, Avi; Goldblum, Micah; Gupta, Arjun; Dickerson, John P.; Goldstein, Tom (2021-07-01). "Just How Toxic is Data Poisoning? A Unified Benchmark for Backdoor and Data Poisoning Attacks" . International Conference on Machine Learning . PMLR: 9389– 9398. ↑ Shan, Shawn; Ding, Wenxin; Passananti, Josephine; Wu, Stanley; Zheng, Haitao; Zhao, Ben Y. (2023). "Nightshade: Prompt-Specific Poisoning Attacks on Text-to-Image Generative Models". arXiv : 2310.13828 [ cs.CR ]. ↑ Biggio, B.; Nelson, B.; Laskov, P. (2011). "Support vector machines under adversarial label noise" . Journal of Machine Learning Research – Proc. 3rd Asian Conf. Machine Learning . 20 : 97– 112. Archived from the original on 2020-08-03. ↑ Kloft, M.; Laskov, P. (2012). "Security analysis of online centroid anomaly detection" (PDF) . Journal of Machine Learning Research . 13 : 3647– 3690. ↑ Rao, Rahul. "AI-Generated Data Can Poison Future AI Models" . Scientific American . Retrieved 2024-06-22 . ↑ Baruch, Gilad; Baruch, Moran; Goldberg, Yoav (2019). "A Little Is Enough: Circumventing Defenses For Distributed Learning" . Advances in Neural Information Processing Systems . 32 . Curran Associates, Inc. arXiv : 1902.06156 . ↑ El-Mhamdi, El-Mahdi; Guerraoui, Rachid; Guirguis, Arsany; Hoang, Lê-Nguyên; Rouault, Sébastien (2022-05-26). "Genuinely distributed Byzantine machine learning" . Distributed Computing . 35 (4): 305– 331. arXiv : 1905.03853 . doi : 10.1007/s00446-022-00427-9 . ISSN 1432-0452 . S2CID 249111966 . ↑ Goldwasser, S.; Kim, Michael P.; Vaikuntanathan, V.; Zamir, Or (2022). "Planting Undetectable Backdoors in Machine Learning Models". arXiv : 2204.06974 [ cs.LG ]. 1 2 Blanchard, Peva; El Mhamdi, El Mahdi; Guerraoui, Rachid; Stainer, Julien (2017). "Machine Learning with Adversaries: Byzantine Tolerant Gradient Descent" . Advances in Neural Information Processing Systems . 30 . Curran Associates, Inc. ↑ Chen, Lingjiao; Wang, Hongyi; Charles, Zachary; Papailiopoulos, Dimitris (2018-07-03). "DRACO: Byzantine-resilient Distributed Training via Redundant Gradients" . International Conference on Machine Learning . PMLR: 903– 912. arXiv : 1803.09877 . ↑ Mhamdi, El Mahdi El; Guerraoui, Rachid; Rouault, Sébastien (2018-07-03). "The Hidden Vulnerability of Distributed Learning in Byzantium" . International Conference on Machine Learning . PMLR: 3521– 3530. arXiv : 1802.07927 . ↑ Allen-Zhu, Zeyuan; Ebrahimianghazani, Faeze; Li, Jerry; Alistarh, Dan (2020-09-28). "Byzantine-Resilient Non-Convex Stochastic Gradient Descent". arXiv : 2012.14368 [ cs.LG ]. Review ↑ Mhamdi, El Mahdi El; Guerraoui, Rachid; Rouault, Sébastien (2020-09-28). Distributed Momentum for Byzantine-resilient Stochastic Gradient Descent . 9th International Conference on Learning Representations (ICLR), May 4–8, 2021 (virtual conference) . Retrieved 2022-10-20 . Review ↑ Data, Deepesh; Diggavi, Suhas (2021-07-01). "Byzantine-Resilient High-Dimensional SGD with Local Iterations on Heterogeneous Data" . International Conference on Machine Learning . PMLR: 2478– 2488. ↑ Karimireddy, Sai Praneeth; He, Lie; Jaggi, Martin (2021-09-29). "Byzantine-Robust Learning on Heterogeneous Datasets via Bucketing". arXiv : 2006.09365 [ cs.LG ]. Review ↑ Nelson, B.; Rubinstein, B. I.; Huang, L.; Joseph, A. D.; Lee, S. J.; Rao, S.; Tygar, J. D. (2012). "Query strategies for evading convex-inducing classifiers" (PDF) . J. Mach. Learn. Res . 13 : 1293– 1332. ↑ "How to steal modern NLP systems with gibberish?" . cleverhans-blog . 2020-04-06 . Retrieved 2020-10-15 . 1 2 3 4 5 6 7 8 Wang, Xinran; Xiang, Yu; Gao, Jun; Ding, Jie (2020-09-13). "Information Laundering for Model Privacy". arXiv : 2009.06112 [ cs.CR ]. 1 2 Dickson, Ben (2021-04-23). "Machine learning: What are membership inference attacks?" . TechTalks . Retrieved 2021-11-07 . 1 2 Goodfellow, Ian J. ; Shlens, Jonathon; Szegedy, Christian (2015). Explaining and Harnessing Adversarial Examples . International Conference on Learning Representations (ICLR). arXiv : 1412.6572 . ↑ Ribeiro, Antonio H.; Zachariah, Dave; Bach, Francis; Schön, Thomas B. (2023). Regularization properties of adversarially-trained linear regression . Thirty-seventh Conference on Neural Information Processing Systems. ↑ Tsipras, Dimitris; Santurkar, Shibani; Engstrom, Logan; Turner, Alexander; Ma, Aleksander (2019). Robustness May Be At Odds with Accuracy . International Conference for Learning Representations. ↑ Dan, C.; Wei, Y.; Ravikumar, P. (2020). Sharp statistical guarantees for adversarially robust Gaussian classification . International Conference on Machine Learning. ↑ Javanmard, A.; Soltanolkotabi, M.; Hassani, H. (2020). Precise tradeoffs in adversarial training for linear regression . Conference on Learning Theory. ↑ Ribeiro, A. H.; Schön, T. B. (2023). "Overparameterized Linear Regression under Adversarial Attacks". IEEE Transactions on Signal Processing . 71 : 601– 614. arXiv : 2204.06274 . Bibcode : 2023ITSP...71..601R . doi : 10.1109/TSP.2023.3246228 . ↑ Yin, D.; Kannan, R.; Bartlett, P. (2019). Rademacher Complexity for Adversarially Robust Generalization . International Conference on Machine Learning. ↑ Ribeiro, Antônio H.; Zachariah, Dave; Bach, Francis; Schön, Thomas B. (2023-10-16), Regularization properties of adversarially-trained linear regression , arXiv : 2310.10807 ↑ Pieter, Huang; Papernot, Sandy; Goodfellow, Nicolas; Duan, Ian; Abbeel, Yan (2017-02-07). Adversarial Attacks on Neural Network Policies . OCLC 1106256905 . ↑ Korkmaz, Ezgi (2022). "Deep Reinforcement Learning Policies Learn Shared Adversarial Features Across MDPs". Thirty-Sixth AAAI Conference on Artificial Intelligence (AAAI-22) . 36 (7): 7229– 7238. arXiv : 2112.09025 . doi : 10.1609/aaai.v36i7.20684 . S2CID 245219157 . ↑ Carlini, Nicholas; Wagner, David (2018). "Audio Adversarial Examples: Targeted Attacks on Speech-to-Text". 2018 IEEE Security and Privacy Workshops (SPW) . pp. 1– 7. arXiv : 1801.01944 . doi : 10.1109/SPW.2018.00009 . ISBN 978-1-5386-8276-0 . S2CID 4475201 . ↑ Alzantot, Moustafa; Balaji, Bharathan; Srivastava, Mani (2017). "Did you hear that? Adversarial Examples Against Automatic Speech Recognition". 31st Conference on Neural Information Processing Systems (NIPS 2017) . arXiv : 1801.00554 . ↑ Rajaratnam, Krishan; Kalita, Jugal (2018). "Noise Flooding for Detecting Audio Adversarial Examples Against Automatic Speech Recognition". 2018 IEEE International Symposium on Signal Processing and Information Technology (ISSPIT) . pp. 197– 201. arXiv : 1812.10061 . doi : 10.1109/ISSPIT.2018.8642623 . ↑ Jagielski, Matthew; Oprea, Alina; Biggio, Battista; Liu, Chang; Nita-Rotaru, Cristina; Li, Bo (May 2018). "Manipulating Machine Learning: Poisoning Attacks and Countermeasures for Regression Learning". 2018 IEEE Symposium on Security and Privacy (SP) . IEEE. pp. 19– 35. arXiv : 1804.00308 . doi : 10.1109/sp.2018.00057 . ISBN 978-1-5386-4353-2 . S2CID 4551073 . ↑ "Attacking Machine Learning with Adversarial Examples" . OpenAI . 2017-02-24 . Retrieved 2020-10-15 . ↑ Gu, Tianyu; Dolan-Gavitt, Brendan; Garg, Siddharth (2019-03-11). "BadNets: Identifying Vulnerabilities in the Machine Learning Model Supply Chain". arXiv : 1708.06733 [ cs.CR ]. ↑ Veale, Michael; Binns, Reuben; Edwards, Lilian (2018-11-28). "Algorithms that remember: model inversion attacks and data protection law" . Philosophical Transactions. Series A, Mathematical, Physical, and Engineering Sciences . 376 (2133). arXiv : 1807.04644 . Bibcode : 2018RSPTA.37680083V . doi : 10.1098/rsta.2018.0083 . ISSN 1364-503X . PMC 6191664 . PMID 30322998 . ↑ Shokri, Reza; Stronati, Marco; Song, Congzheng; Shmatikov, Vitaly (2017-03-31). "Membership Inference Attacks against Machine Learning Models". arXiv : 1610.05820 [ cs.CR ]. 1 2 Goodfellow, Ian J.; Shlens, Jonathon; Szegedy, Christian (2015-03-20). "Explaining and Harnessing Adversarial Examples". arXiv : 1412.6572 [ stat.ML ]. ↑ Madry, Aleksander; Makelov, Aleksandar; Schmidt, Ludwig; Tsipras, Dimitris; Vladu, Adrian (2019-09-04). "Towards Deep Learning Models Resistant to Adversarial Attacks". arXiv : 1706.06083 [ stat.ML ]. ↑ Carlini, Nicholas; Wagner, David (2017-03-22). "Towards Evaluating the Robustness of Neural Networks". arXiv : 1608.04644 [ cs.CR ]. ↑ Brown, Tom B.; Mané, Dandelion; Roy, Aurko; Abadi, Martín; Gilmer, Justin (2018-05-16). "Adversarial Patch". arXiv : 1712.09665 [ cs.CV ]. ↑ Guo, Sensen; Zhao, Jinxiong; Li, Xiaoyu; Duan, Junhong; Mu, Dejun; Jing, Xiao (2021-04-24). "A Black-Box Attack Method against Machine-Learning-Based Anomaly Network Flow Detection Models" . Security and Communication Networks . 2021 . e5578335. doi : 10.1155/2021/5578335 . ISSN 1939-0114 . 1 2 Gomes, Joao (2018-01-17). "Adversarial Attacks and Defences for Convolutional Neural Networks" . Onfido Tech . Retrieved 2021-10-23 . ↑ Guo, Chuan; Gardner, Jacob; You, Yurong; Wilson, Andrew Gordon; Weinberger, Kilian (2019-05-24). "Simple Black-box Adversarial Attacks" . Proceedings of the 36th International Conference on Machine Learning . PMLR: 2484– 2493. arXiv : 1905.07121 . ↑ Weinberger, Kilian (2020). "ML-Retrospectives". On the importance of deconstruction in machine learning research . NeurIPS 2020. 1 2 3 4 5 Andriushchenko, Maksym; Croce, Francesco; Flammarion, Nicolas; Hein, Matthias (2020). "Square Attack: A Query-Efficient Black-Box Adversarial Attack via Random Search" . In Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael (eds.). Computer Vision – ECCV 2020 . Lecture Notes in Computer Science. Vol. 12368. Cham: Springer International Publishing. pp. 484– 501. arXiv : 1912.00049 . doi : 10.1007/978-3-030-58592-1_29 . ISBN 978-3-030-58592-1 . S2CID 208527215 . 1 2 3 4 5 6 7 Chen, Jianbo; Jordan, Michael I.; Wainwright, Martin J. (2019). "HopSkipJumpAttack: A Query-Efficient Decision-Based Attack". arXiv : 1904.02144 [ cs.LG ]. YouTube presentation ↑ Andriushchenko, Maksym; Croce, Francesco; Flammarion, Nicolas; Hein, Matthias (2020-07-29). "Square Attack: a query-efficient black-box adversarial attack via random search". arXiv : 1912.00049 [ cs.LG ]. ↑ "Black-box decision-based attacks on images" . KejiTech . 2020-06-21 . Retrieved 2021-10-25 . 1 2 Goodfellow, Ian J.; Shlens, Jonathon; Szegedy, Christian (2015-03-20). "Explaining and Harnessing Adversarial Examples". arXiv : 1412.6572 [ stat.ML ]. 1 2 "Adversarial example using FGSM | TensorFlow Core" . TensorFlow . Retrieved 2021-10-24 . ↑ Tsui, Ken (2018-08-22). "Perhaps the Simplest Introduction of Adversarial Examples Ever" . Medium . Retrieved 2021-10-24 . ↑ Corona-Figueroa, Abril; Bond-Taylor, Sam; Bhowmik, Neelanjan; Gaus, Yona Falinie A.; Breckon, Toby P.; Shum, Hubert P. H.; Willcocks, Chris G. (2023). Unaligned 2D to 3D Translation with Conditional Vector-Quantized Code Diffusion using Transformers . IEEE/CVF. arXiv : 2308.14152 . 1 2 Carlini, Nicholas; Wagner, David (2017-03-22). "Towards Evaluating the Robustness of Neural Networks". arXiv : 1608.04644 [ cs.CR ]. ↑ "carlini wagner attack" . richardjordan.com . Retrieved 2021-10-23 . ↑ Plotz, Mike (2018-11-26). "Paper Summary: Adversarial Examples Are Not Easily Detected: Bypassing Ten Detection Methods" . Medium . Retrieved 2021-10-23 . ↑ Kishor Datta Gupta; Akhtar, Zahid; Dasgupta, Dipankar (2021). "Determining Sequence of Image Processing Technique (IPT) to Detect Adversarial Attacks". SN Computer Science . 2 (5): 383. arXiv : 2007.00337 . doi : 10.1007/s42979-021-00773-8 . ISSN 2662-995X . S2CID 220281087 . ↑ Dekel, O.; Shamir, O.; Xiao, L. (2010). "Learning to classify with missing and corrupted features" (PDF) . Machine Learning . 81 (2): 149– 178. doi : 10.1007/s10994-009-5124-8 . ↑ Liu, Wei; Chawla, Sanjay (2010). "Mining adversarial patterns via regularized loss minimization" (PDF) . Machine Learning . 81 : 69– 83. doi : 10.1007/s10994-010-5199-2 . S2CID 17497168 . ↑ Biggio, B.; Fumera, G.; Roli, F. (2009). "Evade hard multiple classifier systems" . In Okun, O.; Valentini, G. (eds.). Supervised and Unsupervised Ensemble Methods and Their Applications . Studies in Computational Intelligence. Vol. 245. Berlin / Heidelberg: Springer. pp. 15– 38. Archived from the original on 2015-01-15. ↑ Rubinstein, B. I. P.; Bartlett, P. L.; Huang, L.; Taft, N. (2012). "Learning in a large function space: Privacy-preserving mechanisms for svm learning". Journal of Privacy and Confidentiality . 4 (1): 65– 100. arXiv : 0911.5708 . doi : 10.29012/jpc.v4i1.612 . ↑ Blum, Avrim; Hardt, Moritz (2015-02-16). "The Ladder: A Reliable Leaderboard for Machine Learning Competitions". arXiv : 1502.04585 [ cs.LG ]. ↑ Kantarcioglu, M.; Xi, B.; Clifton, C. (January 2011). "Classifier Evaluation and Attribute Selection against Active Adversaries" (PDF) . Data Min. Knowl. Discov . 22 ( 1– 2): 291– 335. doi : 10.1007/s10618-010-0197-3 . ↑ Chivukula, Aneesh; Yang, Xinghao; Liu, Wei; Zhu, Tianqing; Zhou, Wanlei (2020). "Game Theoretical Adversarial Deep Learning with Variational Adversaries". IEEE Transactions on Knowledge and Data Engineering . 33 (11): 3568– 3581. doi : 10.1109/TKDE.2020.2972320 . hdl : 10453/145751 . ISSN 1558-2191 . S2CID 213845560 . ↑ Chivukula, Aneesh Sreevallabh; Liu, Wei (2019). "Adversarial Deep Learning Models with Multiple Adversaries". IEEE Transactions on Knowledge and Data Engineering . 31 (6): 1066– 1079. Bibcode : 2019ITKDE..31.1066C . doi : 10.1109/TKDE.2018.2851247 . hdl : 10453/136227 . ISSN 1558-2191 . S2CID 67024195 . ↑ "TrojAI" . www.iarpa.gov . Retrieved 2020-10-14 . ↑ Athalye, Anish; Carlini, Nicholas; Wagner, David (2018-02-01). "Obfuscated Gradients Give a False Sense of Security: Circumventing Defenses to Adversarial Example". arXiv : 1802.00420v1 [ cs.LG ]. ↑ He, Warren; Wei, James; Chen, Xinyun; Carlini, Nicholas; Song, Dawn (2017-06-15). "Adversarial Example Defenses: Ensembles of Weak Defenses are not Strong". arXiv : 1706.04701 [ cs.LG ]. ↑ Yerlikaya, Fahri Anıl; Bahtiyar, Şerif (2022-07-14). "Data poisoning attacks against machine learning algorithms" . Expert Systems with Applications . 208 118101. doi : 10.1016/j.eswa.2022.118101 – via Elsevier Science Direct. ↑ Chhabra, Anshuman; Mohapatra, Prasant (2021-11-15). "Moving Target Defense against Adversarial Machine Learning" . Proceedings of the 8th ACM Workshop on Moving Target Defense . MTD '21. New York, NY, USA: Association for Computing Machinery. pp. 29– 30. doi : 10.1145/3474370.3485662 . ISBN 978-1-4503-8658-6 . External links [ edit ] MITRE ATLAS: Adversarial Threat Landscape for Artificial-Intelligence Systems NIST 8269 Draft: A Taxonomy and Terminology of Adversarial Machine Learning Laskov, Pavel; Lippmann, Richard (2010). "Machine learning in adversarial environments". Machine Learning . 81 (2): 115– 119. doi : 10.1007/s10994-010-5207-6 . S2CID 12567278 . Workshop on Artificial Intelligence and Security , (AISec) Series v t e Artificial intelligence (AI) History timeline Glossary Companies Projects List of open-source AI software Concepts Automated reasoning Parameter Hyperparameter Loss functions Regression Bias–variance tradeoff Double descent Overfitting Clustering Gradient descent SGD Quasi-Newton method Conjugate gradient method Backpropagation Attention Convolution Normalization Batchnorm Activation Softmax Sigmoid Rectifier Gating Weight initialization Regularization Datasets Augmentation Prompt engineering Reinforcement learning Q-learning SARSA Imitation Policy gradient Diffusion Latent diffusion model Autoregression Adversary RAG Uncanny valley RLHF Self-supervised learning Reflection Recursive self-improvement Hallucination Word embedding Vibe coding Symbolic AI Neuro-symbolic AI Applications Automated theorem proving Machine learning In-context learning Artificial neural network Deep learning Language model Large NMT Reasoning Model Context Protocol Intelligent agent AI agent Artificial human companion Humanity's Last Exam Lethal autonomous weapons (LAWs) Generative AI Weak AI Hypothetical Artificial general intelligence (AGI) Artificial superintelligence (ASI) Agent2Agent protocol Implementations Audio–visual AlexNet WaveNet Human image synthesis HWR OCR Computer vision Speech synthesis 15.ai ElevenLabs Speech recognition Whisper Facial recognition AlphaFold Text-to-image models Aurora DALL-E Firefly Flux GPT Image Ideogram Imagen Midjourney Recraft Stable Diffusion Text-to-video models Dream Machine Runway Gen Hailuo AI Kling Sora Seedance Veo Music generation Riffusion Suno Udio World models Genie Oasis Text List of large language models Project Debater IBM Watson IBM Watsonx Decisional AlphaGo AlphaZero OpenAI Five Self-driving car MuZero Action selection AutoGPT Robot control Reasoning systems Deductive classifiers Expert systems Inference engines Knowledge-based systems Logic programs Procedural reasoning systems Semantic reasoners Rule-based systems Cognitive architectures ACT-R Soar CLARION LIDA OpenCog Knowledge bases ConceptNet Wikidata DBpedia YAGO People Alan Turing Warren Sturgis McCulloch Walter Pitts John von Neumann Christopher D. Manning Claude Shannon Shun'ichi Amari Kunihiko Fukushima Takeo Kanade Marvin Minsky John McCarthy Nathaniel Rochester Allen Newell Cliff Shaw Herbert A. Simon Oliver Selfridge Frank Rosenblatt Bernard Widrow Joseph Weizenbaum Seymour Papert Seppo Linnainmaa Paul Werbos Geoffrey Hinton John Hopfield Jürgen Schmidhuber Yann LeCun Yoshua Bengio Lotfi A. Zadeh Stephen Grossberg Alex Graves James Goodnight Andrew Ng Fei-Fei Li Alex Krizhevsky Ilya Sutskever Oriol Vinyals Quoc V. Le Ian Goodfellow Demis Hassabis David Silver Andrej Karpathy Ashish Vaswani Noam Shazeer Aidan Gomez John Schulman Mustafa Suleyman Jan Leike Daniel Kokotajlo François Chollet Neural network architectures Neural Turing machine Differentiable neural computer Transformer Vision transformer (ViT) Recurrent neural network (RNN) Long short-term memory (LSTM) Gated recurrent unit (GRU) Echo state network Multilayer perceptron (MLP) Convolutional neural network (CNN) Residual neural network (RNN) Highway network Mamba Autoencoder Variational autoencoder (VAE) Generative adversarial network (GAN) Graph neural network (GNN) Political AI Cold War AI safety ( Alignment ) AI takeover Elections Ethics of AI EU AI Act Nationalism Precautionary principle Regulation of AI US Virtual politician Propaganda Social and economic AI boom AI bubble AI data center AI effect AI literacy AI slop AI winter Anthropomorphism Arms race Competition Environmental impact Explainable AI Generative engine optimization In architecture In education In fiction In healthcare Chatbot psychosis In marketing In video games In visual art Military applications AI warfare Workplace impact Category Retrieved from " https://en.wikipedia.org/w/index.php?title=Adversarial_machine_learning&oldid=1361192819 " Categories : Machine learning Computer security AI safety Hidden categories: Articles with short description Short description is different from Wikidata All articles with dead external links Articles with dead external links from February 2022 Articles with permanently dead external links Wikipedia articles needing clarification from December 2025 This page was last edit … [truncated]