elife04333.xml

<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD v1.1d1 20130915//EN" "JATS-archivearticle1.dtd"><article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="1.1d1"><front><journal-meta><journal-id journal-id-type="nlm-ta">elife</journal-id><journal-id journal-id-type="hwp">eLife</journal-id><journal-id journal-id-type="publisher-id">eLife</journal-id><journal-title-group><journal-title>eLife</journal-title></journal-title-group><issn publication-format="electronic">2050-084X</issn><publisher><publisher-name>eLife Sciences Publications, Ltd</publisher-name></publisher></journal-meta><article-meta><article-id pub-id-type="publisher-id">04333</article-id><article-id pub-id-type="doi">10.7554/eLife.04333</article-id><article-categories><subj-group subj-group-type="display-channel"><subject>Feature article</subject></subj-group><subj-group subj-group-type="heading"><subject>Human biology and medicine</subject></subj-group><subj-group subj-group-type="sub-display-channel"><subject>Science forum</subject></subj-group></article-categories><title-group><article-title>An open investigation of the reproducibility of cancer biology research</article-title></title-group><contrib-group><contrib contrib-type="author" corresp="yes" id="author-16276"><name><surname>Errington</surname><given-names>Timothy M</given-names></name><contrib-id contrib-id-type="orcid">http://orcid.org/0000-0002-4959-5143</contrib-id><xref ref-type="aff" rid="aff1"/><xref ref-type="corresp" rid="cor1">&#x2a;</xref><xref ref-type="other" rid="par-1"/><xref ref-type="fn" rid="con1"/><xref ref-type="fn" rid="conf4"/></contrib><contrib contrib-type="author" id="author-16376"><name><surname>Iorns</surname><given-names>Elizabeth</given-names></name><xref ref-type="aff" rid="aff2"/><xref ref-type="other" rid="par-1"/><xref ref-type="fn" rid="con2"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-16391"><name><surname>Gunn</surname><given-names>William</given-names></name><xref ref-type="aff" rid="aff3"/><xref ref-type="other" rid="par-1"/><xref ref-type="fn" rid="con3"/><xref ref-type="fn" rid="conf4"/></contrib><contrib contrib-type="author" id="author-16378"><name><surname>Tan</surname><given-names>Fraser Elisabeth</given-names></name><xref ref-type="aff" rid="aff4"/><xref ref-type="other" rid="par-1"/><xref ref-type="fn" rid="con4"/><xref ref-type="fn" rid="conf2"/></contrib><contrib contrib-type="author" id="author-16379"><name><surname>Lomax</surname><given-names>Joelle</given-names></name><xref ref-type="aff" rid="aff5"/><xref ref-type="other" rid="par-1"/><xref ref-type="fn" rid="con5"/><xref ref-type="fn" rid="conf3"/></contrib><contrib contrib-type="author" corresp="yes"><name><surname>Nosek</surname><given-names>Brian A</given-names></name><xref ref-type="aff" rid="aff6"/><xref ref-type="corresp" rid="cor2">&#x2a;</xref><xref ref-type="other" rid="par-1"/><xref ref-type="fn" rid="con6"/><xref ref-type="fn" rid="conf4"/></contrib><aff id="aff1"><bold>Timothy M Errington</bold> <institution>Center for Open Science</institution>, <addr-line><named-content content-type="city">Charlottesville</named-content></addr-line>, <country>United States</country></aff><aff id="aff2"><bold>Elizabeth Iorns</bold> <institution>Science Exchange</institution>, <addr-line><named-content content-type="city">Palo Alto</named-content></addr-line>, <country>United States</country></aff><aff id="aff4"><bold>Fraser Elisabeth Tan</bold> <institution>Science Exchange</institution>, <addr-line><named-content content-type="city">Palo Alto</named-content></addr-line>, <country>United States</country></aff><aff id="aff5"><bold>Joelle Lomax</bold> <institution>Science Exchange</institution>, <addr-line><named-content content-type="city">Palo Alto</named-content></addr-line>, <country>United States</country></aff><aff id="aff3"><bold>William Gunn</bold> <institution>Mendeley</institution>, <addr-line><named-content content-type="city">London</named-content></addr-line>, <country>United Kingdom</country></aff><aff id="aff6"><bold>Brian A Nosek</bold> <institution>University of Virginia</institution>, <addr-line><named-content content-type="city">Charlottesville</named-content></addr-line>, <country>United States</country>; <institution>Center for Open Science</institution>, <addr-line><named-content content-type="city">Charlottesville</named-content></addr-line>, <country>United States</country></aff></contrib-group><contrib-group content-type="section"><contrib contrib-type="editor"><name><surname>Rodgers</surname><given-names>Peter</given-names></name><role>Reviewing editor</role><aff><institution>eLife</institution>, <country>United Kingdom</country></aff></contrib></contrib-group><author-notes><corresp id="cor1"><label>&#x2a;</label>For correspondence: <email>tim@cos.io</email> (TME); </corresp><corresp id="cor2"><email>nosek@virginia.edu</email> (BAN)</corresp></author-notes><pub-date publication-format="electronic" date-type="pub"><day>10</day><month>12</month><year>2014</year></pub-date><pub-date pub-type="collection"><year>2014</year></pub-date><volume>3</volume><elocation-id>e04333</elocation-id><permissions><copyright-statement>Copyright &#xa9; 2014, Errington et al</copyright-statement><copyright-year>2014</copyright-year><copyright-holder>Errington et al</copyright-holder><license xlink:href="http://creativecommons.org/licenses/by/4.0/"><license-p>This article is distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution License</ext-link>, which permits unrestricted use and redistribution provided that the original author and source are credited.</license-p></license></permissions><self-uri content-type="pdf" xlink:href="elife04333.pdf"/><abstract><object-id pub-id-type="doi">10.7554/eLife.04333.001</object-id><p>It is widely believed that research that builds upon previously published findings has reproduced the original work. However, it is rare for researchers to perform or publish direct replications of existing results. The Reproducibility Project: Cancer Biology is an open investigation of reproducibility in preclinical cancer biology research. We have identified 50 high impact cancer biology articles published in the period 2010-2012, and plan to replicate a subset of experimental results from each article. A Registered Report detailing the proposed experimental designs and protocols for each subset of experiments will be peer reviewed and published prior to data collection. The results of these experiments will then be published in a Replication Study. The resulting open methodology and dataset will provide evidence about the reproducibility of high-impact results, and an opportunity to identify predictors of reproducibility.</p><p><bold>DOI:</bold> <ext-link ext-link-type="doi" xlink:href="10.7554/eLife.04333.001">http://dx.doi.org/10.7554/eLife.04333.001</ext-link></p></abstract><kwd-group kwd-group-type="author-keywords"><title>Author keywords</title><kwd>reproducibility project: cancer biology</kwd><kwd>methodology</kwd><kwd>open science</kwd><kwd>reproducibility</kwd><kwd>replication</kwd></kwd-group><kwd-group kwd-group-type="research-organism"><title>Research organism</title><kwd>human</kwd><kwd>Mouse</kwd></kwd-group><funding-group><award-group id="par-1"><funding-source><institution-wrap><institution content-type="university">Laura and John Arnold Foundation</institution></institution-wrap></funding-source><principal-award-recipient><name><surname>Errington</surname><given-names>Timothy M</given-names></name><name><surname>Iorns</surname><given-names>Elizabeth</given-names></name><name><surname>Gunn</surname><given-names>William</given-names></name><name><surname>Tan</surname><given-names>Fraser Elisabeth</given-names></name><name><surname>Lomax</surname><given-names>Joelle</given-names></name><name><surname>Nosek</surname><given-names>Brian A</given-names></name></principal-award-recipient></award-group><funding-statement>The Reproducibility Project: Cancer Biology is funded by the Laura and John Arnold Foundation, provided to the Center for Open Science in collaboration with Science Exchange. The funder had no role in study design or the decision to submit the work for publication.</funding-statement></funding-group><custom-meta-group><custom-meta><meta-name>elife-xml-version</meta-name><meta-value>2.0</meta-value></custom-meta><custom-meta specific-use="meta-only"><meta-name>Author impact statement</meta-name><meta-value>The Reproducibility Project: Cancer Biology will generate a high-quality dataset to explore questions about the reproducibility of research, and will make all data, analysis and other research materials openly available to the research community.</meta-value></custom-meta></custom-meta-group></article-meta></front><body><p>Two central features of science are transparency and reproducibility (<xref ref-type="bibr" rid="bib3">Bacon, 1267/1859</xref>; <xref ref-type="bibr" rid="bib23">Jasny et al., 2011</xref>; <xref ref-type="bibr" rid="bib24">Kuhn, 1962</xref>; <xref ref-type="bibr" rid="bib28">Merton, 1942</xref>; <xref ref-type="bibr" rid="bib37">Popper, 1934/1992</xref>). Transparency requires scientists to publish their methodology and data so that the merit of a claim can be assessed on the basis of the evidence rather than the reputation of those making the claim. Reproducibility can refer to both the ability of others to reproduce the findings, given the original data, and to the generation of new data that supports the same conclusions. The focus of this article and project is on the latter&#x2014;the replication of prior results with new data.</p><p>Despite being a defining feature of science, reproducibility is more an assumption than a practice in the present scientific ecosystem (<xref ref-type="bibr" rid="bib10">Collins, 1985</xref>; <xref ref-type="bibr" rid="bib41">Schmidt, 2009</xref>). Incentives for scientific achievement prioritize innovation over replication (<xref ref-type="bibr" rid="bib1">Alberts et al., 2014</xref>; <xref ref-type="bibr" rid="bib33">Nosek, et al., 2012</xref>). Peer review tends to favor manuscripts that contain new findings over those that improve our understanding of a previously published finding. Moreover, careers are made by producing exciting new results at the frontiers of knowledge, not by verifying prior discoveries.<fig id="fig1" position="float"><object-id pub-id-type="doi">10.7554/eLife.04333.002</object-id><label>Figure 1.</label><caption><title>The Reproducibility Project: Cancer Biology will replicate selected experiments from a set of 50 research papers in an effort to estimate the rate of reproducibility in preclinical cancer biology research.</title><p><bold>DOI:</bold> <ext-link ext-link-type="doi" xlink:href="10.7554/eLife.04333.002">http://dx.doi.org/10.7554/eLife.04333.002</ext-link></p></caption><graphic xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="elife04333f001"/><attrib>ILLUSTRATION: <ext-link ext-link-type="uri" xlink:href="http://www.claudiastocker.com">www.claudiastocker.com</ext-link></attrib></fig></p><p>Reproducing prior results is challenging because of insufficient, incomplete, or inaccurate reporting of methodologies (<xref ref-type="bibr" rid="bib18">Hess, 2011</xref>; <xref ref-type="bibr" rid="bib38">Prinz et al., 2011</xref>; <xref ref-type="bibr" rid="bib44">Steward et al., 2012</xref>; <xref ref-type="bibr" rid="bib16">Hackam and Redelmeier, 2006</xref>; <xref ref-type="bibr" rid="bib25">Landis et al., 2011</xref>). Further, a lack of information about research resources makes it difficult or impossible to determine what was used in a published study (<xref ref-type="bibr" rid="bib46">Vasilevsky et al., 2013</xref>). These challenges are compounded by the lack of funding support available from agencies and foundations to support replication research. When replications are performed, they are rarely published (<xref ref-type="bibr" rid="bib10">Collins, 1985</xref>; <xref ref-type="bibr" rid="bib41">Schmidt, 2009</xref>). A literature review in psychological science, for example, estimated that 0.15% of the published results were direct replications of prior published results (<xref ref-type="bibr" rid="bib27">Makel et al., 2012</xref>). Finally, reproducing analyses with prior data is difficult because researchers are often reluctant to share data, even when required by funding bodies or scientific societies (<xref ref-type="bibr" rid="bib50">Wicherts et al., 2006</xref>), and because data loss increases rapidly with time after publication (<xref ref-type="bibr" rid="bib47">Vines et al., 2014</xref>).</p><p>If all published results were true and their effect sizes estimated precisely, then a singular focus on innovation over verification might be inconsequential, because the effect size (which is a measure of the strength of the association between variables, or the magnitude of the differences between experimental conditions) would be reliable. In such a context, the most efficient means of knowledge accumulation would be to spend all resources on discovery and trust that each published result provided an accurate estimate of effects on which to build or extend. However, if not all published results are true and if effect sizes are misestimated, then an absence of replication and verification will lead to a published literature that misrepresents reality. The consequences of that scenario would depend on the magnitude of the misestimation.</p><p>The accumulating evidence suggests an alarming degree of misestimation. Across disciplines, most published studies demonstrate positive results&#x2014;results that indicate an expected association between variables or a difference between experimental conditions (<xref ref-type="bibr" rid="bib43">Sterling, 1959</xref>; <xref ref-type="bibr" rid="bib11">Fanelli, 2010</xref>, <xref ref-type="bibr" rid="bib12">2012</xref>). Simultaneously, across disciplines, the average power of studies to detect positive results is quite low (<xref ref-type="bibr" rid="bib8">Cohen, 1962</xref>; <xref ref-type="bibr" rid="bib20">Ioannidis, 2005</xref>; <xref ref-type="bibr" rid="bib6">Button et al., 2013</xref>). In neuroscience, for example, Button et al. observed the median power of studies to be 21% (<xref ref-type="bibr" rid="bib6">Button et al., 2013</xref>), which means that assuming the finding being investigated is true and accurately estimated, then only 21 of every 100 studies investigating that effect would detect statistically significant evidence for the effect. Most studies would miss detecting the true effect.</p><p>The implication of very low power is that the research literature would be filled with lots of negative results, regardless of whether the effects actually exist or not. In the case of neuroscience, assuming all investigated effects in the published literature are true, only 21% of the studies should have obtained a significant, positive result detecting that effect. However, Fanelli observed a positive result rate of 85% in neuroscience (<xref ref-type="bibr" rid="bib11">Fanelli, 2010</xref>). The samples in these two studies were not the same, but both were large and there is little reason to expect lack of comparability. This discrepancy between observed power and observed positive results is not statistically possible. Instead, it suggests systematic exclusion of negative results (<xref ref-type="bibr" rid="bib15">Greenwald, 1975</xref>) and possibly the exaggeration of positive results by employing flexibility in analytic and reporting practices that inflate the likelihood of false positives (<xref ref-type="bibr" rid="bib42">Simmons et al., 2011</xref>).</p><p>The small amount of direct evidence about reproducibility converges with the conclusions of these systematic reviews. A survey of faculty and trainees at the MD Anderson Cancer Center found half of those researchers reported an inability to reproduce data on at least one occasion (<xref ref-type="bibr" rid="bib29">Mobley et al., 2013</xref>). More dramatically, two industrial laboratories, Bayer and Amgen, reported reproducibility rates of 11% and 25% in two independent efforts to reproduce findings from dozens of groundbreaking basic science studies in oncology and related areas (<xref ref-type="bibr" rid="bib4">Begley and Ellis, 2011</xref>; <xref ref-type="bibr" rid="bib38">Prinz et al., 2011</xref>).</p><p>The available evidence suggests that published research is less reproducible than assumed and desired, perhaps because of an inflation of false positives and a culture of incentives that values publication over accuracy (<xref ref-type="bibr" rid="bib33">Nosek et al., 2012</xref>), but, the evidence is incomplete. The Bayer and Amgen reports of failing to reproduce a high proportion of results provide the most direct evidence. However, neither report made available the effects investigated, the sampling process, the methodology, or the data that comprised the replication efforts (<xref ref-type="bibr" rid="bib30">Nature, 2012</xref>).</p><p>It is vitally important to obtain transparent evidence about the reproducibility of scientific research. To that end, this article introduces the <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="https://osf.io/e81xl">Reproducibility Project: Cancer Biology</ext-link>. This project will conduct replications of findings from 50 high-impact articles in the field of cancer biology published between 2010 and 2012. We selected cancer biology as the area of focus because of the Bayer and Amgen reports, and because of the direct importance of efficient progress in this research discipline for the treatment of disease. The project follows a protocol similar to the Reproducibility Project: Psychology (<xref ref-type="bibr" rid="bib34">Open Science Collaboration, 2012</xref>, <xref ref-type="bibr" rid="bib35">2014</xref>), and&#x2014;in conjunction with <italic>eLife</italic>&#x2014;adopts an approach in which the proposed experiments and protocols are written up in a Registered Report that is peer reviewed and published prior to data collection (<xref ref-type="bibr" rid="bib7">Chambers et al., 2014</xref>; <xref ref-type="bibr" rid="bib32">Nosek and Lakens, 2014</xref>). Following completion of data collection and analysis for each study, the results of the experiments detailed in the Registered Report are published in a Replication Study.</p><sec id="s1"><title>The Reproducibility Project: Cancer Biology</title><p>Replicating a large number of randomly selected studies is the best approach to obtain an estimate of the rate of reproducibility. However, the current incentive structures that strongly favor innovation over replication mean that it is not in the professional interest of any individual scientist or laboratory to conduct and publish replications, particularly many replications. The Reproducibility Project: Cancer Biology circumvents these barriers by establishing a core team to design, prepare, and monitor project operations, and by spreading the data collection effort across multiple researchers and laboratories.</p><sec id="s1-1"><title>Selecting experiments for replication</title><p>Resources are finite. Replication is not needed for findings that have no impact, but it can be of substantial value for increasing the confidence and precision of effects that have substantial impact. We identified a sampling frame that balanced breadth of sampling for general inference with sensible investment of resources on replication projects.</p><p>The sampling frame was defined as the 400 most cited papers from both Scopus and Web of Science using the search terms (cancer, onco&#x2a;, tumor&#x2a;, metasta&#x2a;, neoplas&#x2a;, malignan&#x2a;, carcino&#x2a;) for 2010, 2011, and 2012. Citations were counted from all sources, which include primary research articles and reviews. This produced an initial sample of 501 articles from 2010, 444 from 2011, and 438 from 2012. Altmetrics scores from Mendeley and <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="http://Altmetric.com">Altmetric.com</ext-link> were collected for the entire dataset and used to create a final impact score for each paper. Citation rates and altmetric scores were each standardized by dividing each metric by the highest in the dataset to give each paper a normalized metric score between 0 and 1, which was summed to create an aggregate impact score. Within each year, articles were reviewed for inclusion eligibility starting with the highest aggregate impact article. Articles were removed if they were clinical trials, case studies, reviews, or if they required specialized samples, techniques, or equipment that would be difficult or impossible to obtain. Also, articles reporting sequencing results, such as publications from <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="http://cancergenome.nih.gov">The Cancer Genome Atlas</ext-link> project, were excluded. However, if sequencing or proteomic experiments were only part of an article, the other experiments in those papers could still be eligible. Review of articles continued until a total of 50 articles, about one-third from each year, were identified as eligible. The final set included 17 papers from 2010, 17 from 2011, and 16 from 2012. From each paper, a subset of experiments were identified for replication, prioritizing those that support the main conclusions of the paper while also attending to feasibility and resource constraints. Details on the selection process and a list of the selected and excluded papers are available at the <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="https://osf.io/e81xl/wiki/studies">Open Science Framework</ext-link>.</p><p>There are a variety of alternative sampling strategies that could be pursued in parallel efforts such as community nomination of findings that are important to replicate, request from authors to have their published findings replicated (e.g., the <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="http://validation.scienceexchange.com/">Reproducibility Initiative project</ext-link>), or selection of a sample from a particular journal or on a specific topic for focused investigation. The present sampling strategy focuses replication efforts on high impact papers. This could limit the generalizability of inference to all cancer biology research, but has the benefit of increasing precision and attention to the research that is shaping the field.</p></sec><sec id="s1-2"><title>Preparing and conducting the replications</title><p>The replication experiments are being coordinated by a core group of researchers and conducted by research providers from the Science Exchange network. Because the network consists of over 900 labs skilled in the techniques necessary for replicating the experiments within the chosen studies, the likelihood of a failed replication due to lack of relevant expertise is minimized. The providers are matched to an experiment on the basis of their skills and available instrumentation, often with multiple providers contributing to each replication. An advantage of these labs - commercial contract research organizations (CROs) and core facilities&#x2014;is that they are less likely to be biased for or against replicating the effect. This may reduce the effect of experimenter expectations on observed results (<xref ref-type="bibr" rid="bib40">Rosenthal and Fode, 1963</xref>). However, it does not necessarily eliminate expectancy effects as the replicating researchers are aware of the original findings, and they may have expectations about whether the same result is likely to be obtained or not. Another advantage of this approach is that the time and cost of replicating an experiment via Science Exchange is less than that required to establish a collaboration with another academic lab, allowing the project to scale up efficiently.</p><p>A community of volunteers, largely composed of postdocs in the life sciences, contributed to the project by extracting information from the original papers and drafting protocols for replication experiments. Information about the project including its coordination, planning, execution, and ultimately the replication data is available publicly at the <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="https://osf.io/e81xl/wiki/home/">Open Science Framework</ext-link>. Conducting the project in an open manner increases the accountability, and ideally, the quality of the project and the replications.</p><p>A standardized procedure is followed to minimize irrelevant variation between each replication and to maximize the quality of the replication efforts. We aim to conduct the experimental procedure as closely as possible to the original experiment using the same materials and instrumentation, if available. The replication protocol requires the core team to contact the original corresponding author to request materials and any available information that could improve the quality of the replication attempt.</p><p>Each replication experiment must have high statistical power (1-&#xdf; &#x2265; 0.80) for observing the original effect size in order to minimize the likelihood of failing to replicate because of low power (i.e., a false negative). However, it is common in biomedical research for some experiments to be presented with representative images or graphs without any inferential testing. In these cases, we will inquire with the original authors if there are additional unpublished replications, if it is not already stated in the article, and if any details are available about the results. Further, these qualitative experiments will be replicated three times and all results will be presented. Because the original representative image presents a mean, but no variability information from sampling, it is not possible to compute a standardized effect size. Using the mean of the replication to calculate post-hoc power would be invalid for computing the needed effect size, but the variability of the three replications may be a less biased estimator. So, to determine if more than three replications are needed, we will use the original experiment for estimating the mean and the replications for estimating the variability needed to compute power. This strategy will provide an opportunity to identify the need for more sampling than the default of three. As such, it can only result in increasing the overall power of the investigation.</p><p>An easy way to fail to replicate a result is to do a terrible job of implementing the Materials and methods or conducting the data analysis. Our priority is to maximize the quality of each replication to adequately test the research question. We do this by conducting the entire project transparently so that error in sampling, design, data collection, and analysis can be identified. Moreover, replications describe in detail the entire experimental design, including controls, conditions, assay optimizations, materials, protocols, and analysis plans prior to initiating data collection. Next, a key part of matching experiments with laboratories is to identify labs with the appropriate expertise to maximize research quality. This is particularly important with new and innovative techniques, though most techniques called for in the selected experiments are standard techniques for which expertise is widely available. As experiments are matched to labs, it is possible that no appropriate service provider can be identified. If appropriate expertise is not available, then the finding or paper will be excluded from the project.</p><p>Once the experimental designs and protocols are prepared, the core team solicits feedback from the original authors to identify ways to improve the design. Author input is incorporated into the designs and protocols prior to data collection; suggestions or concerns from the original authors that are not implemented are recorded in the Registered Report. The replication team will also conduct a literature review for evidence of existing replications although, as noted above, direct replication is likely to be rare. Existing published evidence for replication might indicate the likelihood of reproducing the original results.</p></sec><sec id="s1-3"><title>Registered Reports</title><p>In addition to the informal information exchange between the core team and the original authors, each study will undergo peer review prior to data collection following the Registered Report format (<xref ref-type="bibr" rid="bib7">Chambers et al., 2014</xref>; <xref ref-type="bibr" rid="bib32">Nosek and Lakens, 2014</xref>). Peer reviewers at <italic>eLife,</italic> including subject experts and a statistician, will evaluate the appropriateness and quality of the experimental designs and protocols for replication, as described in the Registered Report. If the Registered Report passes peer review, it will be published prior to data collection. Publication of the Replication Study is then contingent on the replication team following through with the approved design, data collection, and analysis plan; publication of the Replication Study is not contingent on the results. This places the incentives for the replication team and the reviewers on maximizing the quality of methodology and minimizes incentives for achieving a specific result.</p><p>The accepted experimental designs and protocols described in the Registered Report will be preregistered publicly at the <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:href="http://osf.io/">Open Science Framework</ext-link> and the Registered Report for each study will be published by <italic>eLife</italic> before any experiments are performed. Following completion of data collection and analysis the Replication Study will be published by <italic>eLife</italic> with all data, analysis scripts, reports, and other research materials added to the project on the Open Science Framework for the research community to view, critique, or extend. The collected body of evidence will be the largest public dataset for investigating reproducibility in cancer biology.</p></sec></sec><sec id="s2"><title>Evaluation of reproducibility</title><p>What is a successful replication? A seemingly easy answer is that the replication produces the same result as the original. However, few results are easily categorized as either the same or different. One approach is to consider whether the replication achieves a p-value of less than 0.05 with the same direction of association or ordinal ranking between conditions as the original. This provides some information but still treats experimental outputs dichotomously, with an arbitrary significance threshold. For example, the above approach would classify a replication with p &#x3d; 0.06 as a failure. An alternative approach in the null hypothesis significance testing framework is to treat the original effect size as the null and test whether the replication is significantly different from that value. This provides complementary information to the first approach because effects can succeed or fail on one or both tests. For example, a replication with p &#x3d; 0.06 fails the first test but may not be distinguishable from the original on the second test.</p><p>Another approach is to compare the effect sizes of the original and replication studies and to then compare whether the estimates are within each other's confidence intervals. This starts to move the inference process away from dichotomous classification and toward estimating effect magnitude and precision of estimation. In another approach, the totality of evidence for an effect is represented by the meta-analytic estimate combining the original and new experiments. This combines all evidence and provides an indication of the present knowledge of the effect.</p><p>There is no single answer to the question &#x2018;what is a successful replication?&#x2019; (<xref ref-type="bibr" rid="bib45">Valentine et al., 2011</xref>). As such, we will report multiple indicators of comparison and combination of original and replication effects in order to gain a better understanding of the findings examined and of reproducibility more generally.</p><sec id="s2-1"><title>What will and will not be learned</title><p>The primary goals of this project are to produce an initial estimate of the reproducibility of cancer biology research and to identify predictors of reproducibility. These are big questions for one study to address. Nevertheless, the results will provide an initial empirical basis to evaluate reproducibility and may help guide the broader discussion about reproducibility toward areas of significant challenge, productive areas for further inquiry, and possible interventions for improvement. Given the importance of these questions, it is important to recognize what will and will not be learned from the results of the Reproducibility Project: Cancer Biology.</p></sec><sec id="s2-2"><title>Does a failure to replicate mean that the original result was a false positive?</title><p>No. There are many reasons that two studies of the same phenomenon could obtain different results, and only one of those is that the original was a false positive. The project design minimizes but does not eliminate the possibility of other explanations such as the replication being a false negative due to insufficient power, error in analysis, differences in statistical methods, or error in the design and implementation of the study procedures such as reagent variability/lack of validation, unintentional selective reporting, lack of appropriate controls, lack of equipment calibration, or unrecognized experimental variables (<xref ref-type="bibr" rid="bib20">Ioannidis, 2005</xref>; <xref ref-type="bibr" rid="bib31">Nieuwenhuis et al., 2011</xref>; <xref ref-type="bibr" rid="bib26">Loscalzo, 2012</xref>; <xref ref-type="bibr" rid="bib17">Haibe-Kains et al., 2013</xref>; <xref ref-type="bibr" rid="bib39">Pusztai et al., 2013</xref>; <xref ref-type="bibr" rid="bib46">Vasilevsky et al., 2013</xref>; <xref ref-type="bibr" rid="bib19">Hines et al., 2014</xref>; <xref ref-type="bibr" rid="bib22">Ioannidis et al., 2014</xref>; <xref ref-type="bibr" rid="bib36">Perrin, 2014</xref>).</p><p>Other causes for different research outcomes between original and replication have implications for understanding the phenomenon itself. For instance, the original effect may be real but overestimated by the original study. As such, replication may provide new insight, not of the truth of the effect, but of its practical implications. Another reason for different outcomes is that the conditions necessary to obtain the result are not yet understood. Together the replication protocol and peer review of the Registered Report are intended to produce an experimental design for which there is no reason to expect a priori a different result than the original. Those expectations are based on the present theoretical understanding of how and why the effect occurs. However, that understanding may be incorrect or incomplete. Particular features of the original experimental protocol might be critical but unidentified. Therefore, if there is no reason to expect a different result, and a different result is obtained, then differences between the original and replication deemed previously to be non-consequential are now targets for hypothesizing and investigation. This may produce new discoveries and enhance understanding of the effect, the conditions necessary to obtain it, and its implications for biology. This type of discovery is unlikely to occur without direct replication.</p></sec><sec id="s2-3"><title>Does a successful replication mean that the original interpretation is correct?</title><p>No. Successful direct replication can verify that a result can be obtained and establishes some generalizability by showing that it can be obtained in different circumstances. However, direct replication does not confirm the interpretation of the result. For example, if an original design has an unidentified confounding influence, then the direct replication is also likely to be influenced by that confound.</p><p>Developing understanding for the meaning of research findings is often clarified more productively through conceptual replication (<xref ref-type="bibr" rid="bib41">Schmidt, 2009</xref>). In direct replication the original methodology is reproduced as faithfully as possible; in conceptual replication the original research question is tested again with different methods. Conceptual replication can include changes to the model system used, leveraging a new technology or improved procedure, or an operational change to the manipulation or measurement of critical variables. Such changes are done to remove alternative explanations and demonstrate that the phenomenon is not idiosyncratic to the original procedures. Conceptual replication is as vital for gaining understanding of an effect as direct replication is for increasing confidence that the effect occurs.</p></sec></sec><sec sec-type="conclusion" id="s3"><title>Conclusion</title><p>Replication is central to the progress of science: if others cannot reproduce the evidence backing a scientific claim, then the claim loses status as scientific knowledge. This process differentiates science from other ways of knowing for which the power, authority, ideology, or persuasiveness of the person making the claim determines its truth.</p><p>The Reproducibility Project: Cancer Biology uses an open methodology to examine reproducibility in cancer biology research. The implications of the project may depend on its outcomes. A high rate of reproducibility might affirm current research and reporting practices, which may suggest that the potentially dysfunctional incentives in the present ecosystem are relatively inert (See <xref ref-type="bibr" rid="bib22">Ioannidis et al., 2014</xref> for a review). On the other hand, a low rate of reproducibility might foster changes by researchers, scientific societies, universities, publishers, and funding agencies to improve research practices and to adjust the training and incentives that maintain them (<xref ref-type="bibr" rid="bib21">Ioannidis and Khoury, 2011</xref>; <xref ref-type="bibr" rid="bib25">Landis et al., 2012</xref>; <xref ref-type="bibr" rid="bib28a">Miguel et al., 2014</xref>; <xref ref-type="bibr" rid="bib33">Nosek et al., 2012</xref>; <xref ref-type="bibr" rid="bib49">Wagenmakers et al., 2012</xref>; <xref ref-type="bibr" rid="bib48">Wadmann, 2013</xref>; <xref ref-type="bibr" rid="bib1">Alberts et al., 2014</xref>; <xref ref-type="bibr" rid="bib2">Asendorpf et al., 2013</xref>; <xref ref-type="bibr" rid="bib9">Collins and Tabak, 2014</xref>).</p><p>Self-examination is not without challenge. A low reproducibility estimate might prompt concern that the reputation of cancer biology research will be damaged (<xref ref-type="bibr" rid="bib5">Bissell, 2013</xref>). However, we believe that there is much greater risk in having a low reproducibility rate and failing to discover it. Science can only self-correct if there is awareness of what needs correcting. If reproducibility is much lower than expected, then the generation of new knowledge will suffer because it is difficult to pursue innovation and discovery if the foundation of evidence is not reliable (<xref ref-type="bibr" rid="bib13">Forscher, 1963</xref>). A culture that values and practices reproducible science can push out the boundaries of knowledge with confidence that new discoveries have potential to lead to new knowledge and, in the case of cancer biology, cures to one of the greatest challenges to human health.</p></sec></body><back><ack id="ack"><title>Acknowledgements</title><p>We would like to thank the following companies for generously donating reagents to the Reproducibility Project: Cancer Biology; BioLegend, Charles River Laboratories, Corning Incorporated, DDC Medical, EMD Millipore, Harlan Laboratories, LI-COR Biosciences, Mirus Bio, Novus Biologicals, and Sigma&#x2013;Aldrich.</p></ack><fn-group content-type="competing-interest"><title>Competing interests:</title><fn fn-type="conflict" id="conf1"><p>EI: Employed by and hold shares in Science Exchange Inc</p></fn><fn fn-type="conflict" id="conf2"><p>FET: Employed by and hold shares in Science Exchange Inc</p></fn><fn fn-type="conflict" id="conf3"><p>JL: Employed by and hold shares in Science Exchange Inc.</p></fn><fn fn-type="conflict" id="conf4"><p>The other authors declare that no competing interests exist.</p></fn></fn-group><fn-group content-type="author-contribution"><title>Author contributions</title><fn fn-type="con" id="con1"><p>TME, Conception and design, Drafting or revising the article</p></fn><fn fn-type="con" id="con2"><p>EI, Conception and design, Drafting or revising the article</p></fn><fn fn-type="con" id="con3"><p>WG, Conception and design, Drafting or revising the article</p></fn><fn fn-type="con" id="con4"><p>FET, Conception and design, Drafting or revising the article</p></fn><fn fn-type="con" id="con5"><p>JL, Conception and design, Drafting or revising the article</p></fn><fn fn-type="con" id="con6"><p>BAN, Conception and design, Drafting or revising the article</p></fn></fn-group><ref-list><title>References</title><ref id="bib1"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Alberts</surname><given-names>B</given-names></name><name><surname>Kirschner</surname><given-names>MW</given-names></name><name><surname>Tilghman</surname><given-names>S</given-names></name><name><surname>Varmus</surname><given-names>H</given-names></name></person-group><year>2014</year><article-title>Rescuing US biomedical research from its systemic flaws</article-title><source>Proceedings of the National Academy of Sciences of USA</source><volume>111</volume><fpage>5773</fpage><lpage>5777</lpage><pub-id pub-id-type="doi">10.1073/pnas.1404402111</pub-id></element-citation></ref><ref id="bib2"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Asendorpf</surname><given-names>JB</given-names></name><name><surname>Conner</surname><given-names>M</given-names></name><name><surname>De Fruyt</surname><given-names>F</given-names></name><name><surname>De Houwer</surname><given-names>J</given-names></name><name><surname>Denissen</surname><given-names>JJ</given-names></name><name><surname>Fiedler</surname><given-names>K</given-names></name><name><surname>Fiedler</surname><given-names>S</given-names></name><name><surname>Funder</surname><given-names>DC</given-names></name><name><surname>Kliegl</surname><given-names>R</given-names></name><name><surname>Nosek</surname><given-names>BA</given-names></name><name><surname>Perugini</surname><given-names>M</given-names></name><name><surname>Roberts</surname><given-names>BW</given-names></name><name><surname>Schmitt</surname><given-names>M</given-names></name><name><surname>van Aken</surname><given-names>MA</given-names></name><name><surname>Weber</surname><given-names>H</given-names></name><name><surname>Wicherts</surname><given-names>JM</given-names></name></person-group><year>2013</year><article-title>Recommendations for increasing replicability in psychology</article-title><source>European Journal of Personality</source><volume>27</volume><fpage>108</fpage><lpage>119</lpage><pub-id pub-id-type="doi">10.1002/per.1919</pub-id></element-citation></ref><ref id="bib3"><element-citation publication-type="book"><person-group person-group-type="author"><name><surname>Bacon</surname><given-names>R</given-names></name></person-group><year>1267/1859</year><source>Fr. Rogeri Bacon Opera qu&#xe6;dam hactenus inedita. Vol. I. containing I.&#x2013;Opus tertium. II.&#x2013;Opus minus. III.&#x2013;Compendium philosophi&#xe6;</source><publisher-name>Longman, Green, Longman, Roberts</publisher-name><comment>Retrieved from</comment><ext-link ext-link-type="uri" xlink:href="http://books.google.com/books?id=wMUKAAAAYAAJ">http://books.google.com/books?id&#x3d;wMUKAAAAYAAJ</ext-link></element-citation></ref><ref id="bib4"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Begley</surname><given-names>CG</given-names></name><name><surname>Ellis</surname><given-names>LM</given-names></name></person-group><year>2012</year><article-title>Raise standards for preclinical cancer research</article-title><source>Nature</source><volume>483</volume><fpage>531</fpage><lpage>533</lpage><pub-id pub-id-type="doi">10.1038/483531a</pub-id></element-citation></ref><ref id="bib5"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Bissell</surname><given-names>M</given-names></name></person-group><year>2013</year><article-title>The risks of the replication drive</article-title><source>Nature</source><volume>503</volume><fpage>333</fpage><lpage>334</lpage><pub-id pub-id-type="doi">10.1038/503333a</pub-id></element-citation></ref><ref id="bib6"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Button</surname><given-names>KS</given-names></name><name><surname>Ioannidis</surname><given-names>JP</given-names></name><name><surname>Mokrysz</surname><given-names>C</given-names></name><name><surname>Nosek</surname><given-names>BA</given-names></name><name><surname>Flint</surname><given-names>J</given-names></name><name><surname>Robinson</surname><given-names>ES</given-names></name><name><surname>Munafo</surname><given-names>MR</given-names></name></person-group><year>2013</year><article-title>Power failure: why small sample size undermines the reliability of neuroscience</article-title><source>Nature Reviews Neuroscience</source><volume>14</volume><fpage>365</fpage><lpage>376</lpage><pub-id pub-id-type="doi">10.1038/nrn3475</pub-id></element-citation></ref><ref id="bib7"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Chambers</surname><given-names>CD</given-names></name><name><surname>Feredoes</surname><given-names>E</given-names></name><name><surname>Muthukumaraswamy</surname><given-names>SD</given-names></name><name><surname>Etchells</surname><given-names>PJ</given-names></name></person-group><year>2014</year><article-title>Instead of &#x201c;playing the game&#x201d; it is time to change the rules: Registered Reports at AIMS Neuroscience and beyond</article-title><source>AIMS Neuroscience</source><volume>1</volume><fpage>4</fpage><lpage>17</lpage><pub-id pub-id-type="doi">10.3934/Neuroscience2014.1.4</pub-id></element-citation></ref><ref id="bib8"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Cohen</surname><given-names>J</given-names></name></person-group><year>1962</year><article-title>The statistical power of abnormal-social psychological research: a review</article-title><source>Journal of Abnormal and Social Psychology</source><volume>65</volume><fpage>145</fpage><lpage>153</lpage><pub-id pub-id-type="doi">10.1037/h0045186</pub-id></element-citation></ref><ref id="bib9"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Collins</surname><given-names>FS</given-names></name><name><surname>Tabak</surname><given-names>LA</given-names></name></person-group><year>2014</year><article-title>NIH plans to enhance reproducibility</article-title><source>Nature</source><volume>503</volume><fpage>612</fpage><lpage>613</lpage><pub-id pub-id-type="doi">10.1038/505612a</pub-id></element-citation></ref><ref id="bib10"><element-citation publication-type="book"><person-group person-group-type="author"><name><surname>Collins</surname><given-names>HM</given-names></name></person-group><year>1985</year><source>Changing Order: Replication and Induction in Scientific Practice.</source><publisher-loc>London</publisher-loc><publisher-name>Sage</publisher-name></element-citation></ref><ref id="bib11"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Fanelli</surname><given-names>D</given-names></name></person-group><year>2010</year><article-title>&#x201c;Positive&#x201d; results increase down the Hierarchy of the Sciences</article-title><source>PLOS ONE</source><volume>5</volume><fpage>e10068</fpage><pub-id pub-id-type="doi">10.1371/journal.pone.0010068</pub-id></element-citation></ref><ref id="bib12"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Fanelli</surname><given-names>D</given-names></name></person-group><year>2012</year><article-title>Negative results are disappearing from most disciplines and countries</article-title><source>Scientometrics</source><volume>90</volume><fpage>891</fpage><lpage>904</lpage><pub-id pub-id-type="doi">10.1007/s11192-011-0494-7</pub-id></element-citation></ref><ref id="bib13"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Forscher</surname><given-names>BK</given-names></name></person-group><year>1963</year><article-title>Chaos in the Brickyard</article-title><source>Science</source><volume>142</volume><fpage>339</fpage><pub-id pub-id-type="doi">10.1126/science.142.3590.339</pub-id></element-citation></ref><ref id="bib15"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Greenwald</surname><given-names>AG</given-names></name></person-group><year>1975</year><article-title>Consequences of prejudice against the null hypothesis</article-title><source>Psychological Bulletin</source><volume>82</volume><fpage>1</fpage><lpage>20</lpage><pub-id pub-id-type="doi">10.1037/h0076157</pub-id></element-citation></ref><ref id="bib16"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Hackam</surname><given-names>DG</given-names></name><name><surname>Redelmeier</surname><given-names>DA</given-names></name></person-group><year>2006</year><article-title>Translation of research evidence from animals to humans</article-title><source>Jama</source><volume>296</volume><fpage>1727</fpage><lpage>1732</lpage><pub-id pub-id-type="doi">10.1001/jama.296.14.1731</pub-id></element-citation></ref><ref id="bib17"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Haibe-Kains</surname><given-names>B</given-names></name><name><surname>El-Hachem</surname><given-names>N</given-names></name><name><surname>Birkbak</surname><given-names>NJ</given-names></name><name><surname>Jin</surname><given-names>AC</given-names></name><name><surname>Beck</surname><given-names>AH</given-names></name><name><surname>Aerts</surname><given-names>HJ</given-names></name><name><surname>Quackenbush</surname><given-names>J</given-names></name></person-group><year>2013</year><article-title>Inconsistency in large pharmacogenomic studies</article-title><source>Nature</source><volume>504</volume><fpage>389</fpage><lpage>393</lpage><pub-id pub-id-type="doi">10.1038/nature12831</pub-id></element-citation></ref><ref id="bib18"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Hess</surname><given-names>KR</given-names></name></person-group><year>2011</year><article-title>Statistical design considerations in animal studies published recently in cancer research</article-title><source>Cancer Research</source><volume>71</volume><fpage>625</fpage><pub-id pub-id-type="doi">10.1158/0008-5472.can-10-3296</pub-id></element-citation></ref><ref id="bib19"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Hines</surname><given-names>WC</given-names></name><name><surname>Su</surname><given-names>Y</given-names></name><name><surname>Kuhn</surname><given-names>I</given-names></name><name><surname>Polyak</surname><given-names>K</given-names></name><name><surname>Bissell</surname><given-names>MJ</given-names></name></person-group><year>2014</year><article-title>Sorting out the FACS: a devil in the details</article-title><source>Cell Reports</source><volume>6</volume><fpage>779</fpage><lpage>781</lpage><pub-id pub-id-type="doi">10.1016/j.celrep.2014.02.021</pub-id></element-citation></ref><ref id="bib20"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Ioannidis</surname><given-names>JP</given-names></name></person-group><year>2005</year><article-title>Why most published research findings are false</article-title><source>PLOS Medicine</source><volume>2</volume><fpage>e124</fpage><pub-id pub-id-type="doi">10.1371/journal.pmed.0020124</pub-id></element-citation></ref><ref id="bib21"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Ioannidis</surname><given-names>JP</given-names></name><name><surname>Khoury</surname><given-names>MJ</given-names></name></person-group><year>2011</year><article-title>Improving validation practices in &#x201c;Omics&#x201d; research</article-title><source>Science</source><volume>334</volume><fpage>1230</fpage><lpage>1232</lpage><pub-id pub-id-type="doi">10.1126/science.1211811</pub-id></element-citation></ref><ref id="bib22"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Ioannidis</surname><given-names>JP</given-names></name><name><surname>Munafo</surname><given-names>MR</given-names></name><name><surname>Fusar-Poli</surname><given-names>P</given-names></name><name><surname>Nosek</surname><given-names>BA</given-names></name><name><surname>David</surname><given-names>SP</given-names></name></person-group><year>2014</year><article-title>Publication and other reporting biases in cognitive sciences: Detection, prevalence, and prevention</article-title><source>Trends in Cognitive Sciences</source><volume>18</volume><fpage>235</fpage><lpage>241</lpage><pub-id pub-id-type="doi">10.1016/j.tics.2014.02.010</pub-id></element-citation></ref><ref id="bib23"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Jasny</surname><given-names>BR</given-names></name><name><surname>Chin</surname><given-names>G</given-names></name><name><surname>Chong</surname><given-names>L</given-names></name><name><surname>Vignieri</surname><given-names>S</given-names></name></person-group><year>2011</year><article-title>Data replication &#x26; reproducibility. Again, and again, and again&#x2026;</article-title><source>Science</source><volume>334</volume><fpage>1225</fpage><pub-id pub-id-type="doi">10.1126/science.334.6060.1225</pub-id></element-citation></ref><ref id="bib24"><element-citation publication-type="book"><person-group person-group-type="author"><name><surname>Kuhn</surname><given-names>TS</given-names></name></person-group><year>1962</year><source>The Structure of Scientific Revolutions</source><publisher-loc>Chicago, Illinois</publisher-loc><publisher-name>University of Chicago Press</publisher-name></element-citation></ref><ref id="bib25"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Landis</surname><given-names>SC</given-names></name><name><surname>Amara</surname><given-names>SG</given-names></name><name><surname>Asadullah</surname><given-names>K</given-names></name><name><surname>Austin</surname><given-names>CP</given-names></name><name><surname>Blumenstein</surname><given-names>R</given-names></name><name><surname>Bradley</surname><given-names>EW</given-names></name><name><surname>Cyrstal</surname><given-names>RG</given-names></name><name><surname>Darnell</surname><given-names>RB</given-names></name><name><surname>Ferrante</surname><given-names>RJ</given-names></name><name><surname>Fillit</surname><given-names>H</given-names></name><name><surname>Finkelstein</surname><given-names>R</given-names></name><name><surname>Fisher</surname><given-names>M</given-names></name><name><surname>Gendelman</surname><given-names>HE</given-names></name><name><surname>Golub</surname><given-names>RM</given-names></name><name><surname>Goudreau</surname><given-names>JL</given-names></name><name><surname>Gross</surname><given-names>RA</given-names></name><name><surname>Gubitz</surname><given-names>AK</given-names></name><name><surname>Hesterlee</surname><given-names>SE</given-names></name><name><surname>Howells</surname><given-names>DW</given-names></name><name><surname>Huguenard</surname><given-names>J</given-names></name><name><surname>Kelner</surname><given-names>K</given-names></name><name><surname>Koroshetz</surname><given-names>W</given-names></name><name><surname>Krainc</surname><given-names>D</given-names></name><name><surname>Lazic</surname><given-names>SE</given-names></name><name><surname>Levine</surname><given-names>MS</given-names></name><name><surname>Macleod</surname><given-names>MR</given-names></name><name><surname>McCall</surname><given-names>JM</given-names></name><name><surname>Moxley</surname><given-names>RT</given-names></name><name><surname>Narasimhan</surname><given-names>K</given-names></name><name><surname>Nobel</surname><given-names>LJ</given-names></name><name><surname>Perrin</surname><given-names>S</given-names></name><name><surname>Porter</surname><given-names>JD</given-names></name><name><surname>Steward</surname><given-names>O</given-names></name><name><surname>Unger</surname><given-names>E</given-names></name><name><surname>Utz</surname><given-names>U</given-names></name><name><surname>Silberberg</surname><given-names>SD</given-names></name></person-group><year>2012</year><article-title>A call for transparent reporting to optimize the predictive value of preclinical research</article-title><source>Nature</source><volume>490</volume><fpage>187</fpage><lpage>190</lpage><pub-id pub-id-type="doi">10.1038/nature11556</pub-id></element-citation></ref><ref id="bib26"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Loscalzo</surname><given-names>J</given-names></name></person-group><year>2012</year><article-title>Irreproducible experimental results: causes, (mis)interpretations, and consequences</article-title><source>Circulation</source><volume>125</volume><fpage>1211</fpage><lpage>1214</lpage><pub-id pub-id-type="doi">10.1161/circulationaha.112.098244</pub-id></element-citation></ref><ref id="bib27"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Makel</surname><given-names>MC</given-names></name><name><surname>Plucker</surname><given-names>JA</given-names></name><name><surname>Hagerty</surname><given-names>B</given-names></name></person-group><year>2012</year><article-title>Replications in psychology research: how often do they really occur?</article-title><source>Perspectives on Psychological Science</source><volume>7</volume><fpage>537</fpage><lpage>542</lpage><pub-id pub-id-type="doi">10.1177/1745691612460688</pub-id></element-citation></ref><ref id="bib28"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Merton</surname><given-names>RK</given-names></name></person-group><year>1942</year><article-title>Science and technology in a democratic order</article-title><source>Journal of Legal and Political Sociology</source><volume>1</volume><fpage>115</fpage><lpage>126</lpage></element-citation></ref><ref id="bib28a"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Miguel</surname><given-names>E</given-names></name><name><surname>Camerer</surname><given-names>C</given-names></name><name><surname>Casey</surname><given-names>K</given-names></name><name><surname>Cohen</surname><given-names>J</given-names></name><name><surname>Esterling</surname><given-names>KM</given-names></name><name><surname>Gerber</surname><given-names>A</given-names></name><name><surname>Glennerster</surname><given-names>R</given-names></name><name><surname>Green</surname><given-names>DP</given-names></name><name><surname>Humphreys</surname><given-names>M</given-names></name><name><surname>Imbens</surname><given-names>G</given-names></name><name><surname>Laitin</surname><given-names>D</given-names></name><name><surname>Madon</surname><given-names>T</given-names></name><name><surname>Nelson</surname><given-names>L</given-names></name><name><surname>Nosek</surname><given-names>BA</given-names></name><name><surname>Petersen</surname><given-names>M</given-names></name><name><surname>Sedlmayr</surname><given-names>R</given-names></name><name><surname>Simmons</surname><given-names>JP</given-names></name><name><surname>Simonsohn</surname><given-names>U</given-names></name><name><surname>Van der Laan</surname><given-names>M</given-names></name></person-group><year>2014</year><article-title>Promoting transparency in social science research</article-title><source>Science</source><volume>343</volume><fpage>30</fpage><lpage>31</lpage><pub-id pub-id-type="doi">10.1126/science.1245317</pub-id></element-citation></ref><ref id="bib29"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Mobley</surname><given-names>A</given-names></name><name><surname>Linder</surname><given-names>SK</given-names></name><name><surname>Braeuer</surname><given-names>R</given-names></name><name><surname>Ellis</surname><given-names>LM</given-names></name><name><surname>Zwelling</surname><given-names>L</given-names></name></person-group><year>2013</year><article-title>A survey on data reproducibility in cancer research provides insights into our limited ability to translate findings from the laboratory to the clinic</article-title><source>PLOS ONE</source><volume>8</volume><fpage>e63221</fpage><pub-id pub-id-type="doi">10.1371/journal.pone.0063221</pub-id></element-citation></ref><ref id="bib30"><element-citation publication-type="journal"><person-group person-group-type="author"><collab>Nature</collab></person-group><year>2012</year><article-title>Collateral damage</article-title><source>Nature</source><volume>485</volume><fpage>41</fpage><pub-id pub-id-type="doi">10.1038/485041e</pub-id></element-citation></ref><ref id="bib31"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Nieuwenhuis</surname><given-names>S</given-names></name><name><surname>Forstmann</surname><given-names>BU</given-names></name><name><surname>Wagenmakers</surname><given-names>EJ</given-names></name></person-group><year>2011</year><article-title>Erroneous analyses of interactions in neuroscience: a problem of significance</article-title><source>Nature Neuroscience</source><volume>14</volume><fpage>1105</fpage><lpage>1107</lpage><pub-id pub-id-type="doi">10.1038/nn.2886</pub-id></element-citation></ref><ref id="bib32"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Nosek</surname><given-names>BA</given-names></name><name><surname>Lakens</surname><given-names>D</given-names></name></person-group><year>2014</year><article-title>Registered reports: a method to increase the credibility of published results</article-title><source>Social Psychology</source><volume>45</volume><fpage>137</fpage><lpage>141</lpage><pub-id pub-id-type="doi">10.1027/1864-9335/a000192</pub-id></element-citation></ref><ref id="bib33"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Nosek</surname><given-names>BA</given-names></name><name><surname>Spies</surname><given-names>JR</given-names></name><name><surname>Motyl</surname><given-names>M</given-names></name></person-group><year>2012</year><article-title>Scientific Utopia: II. Restructuring incentives and practices to promote truth over publishability</article-title><source>Perspectives on Psychological Science</source><volume>7</volume><fpage>615</fpage><lpage>631</lpage><pub-id pub-id-type="doi">10.1177/1745691612459058</pub-id></element-citation></ref><ref id="bib34"><element-citation publication-type="journal"><person-group person-group-type="author"><collab>Open Science Collaboration</collab></person-group><year>2012</year><article-title>An open, large-scale, collaborative effort to estimate the reproducibility of psychological science</article-title><source>Perspectives on Psychological Science</source><volume>7</volume><fpage>657</fpage><lpage>660</lpage><pub-id pub-id-type="doi">10.1177/1745691612462588</pub-id></element-citation></ref><ref id="bib35"><element-citation publication-type="book"><person-group person-group-type="author"><collab>Open Science Collaboration</collab></person-group><year>2014</year><article-title>The Reproducibility Project: a model of large-scale collaboration for empirical research on reproducibility</article-title><person-group person-group-type="editor"><name><surname>Stodden</surname><given-names>V</given-names></name><name><surname>Leish</surname><given-names>VF</given-names></name><name><surname>Peng</surname><given-names>R</given-names></name></person-group><source>Implementing Reproducible Computational Research (A Volume in the R Series)</source><publisher-loc>New York</publisher-loc><publisher-name>Taylor &#x26; Francis</publisher-name></element-citation></ref><ref id="bib36"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Perrin</surname><given-names>S</given-names></name></person-group><year>2014</year><article-title>Preclinical research: Make mouse studies work</article-title><source>Nature</source><volume>507</volume><fpage>423</fpage><lpage>425</lpage><pub-id pub-id-type="doi">10.1038/507423a</pub-id></element-citation></ref><ref id="bib37"><element-citation publication-type="book"><person-group person-group-type="author"><name><surname>Popper</surname><given-names>K</given-names></name></person-group><year>1934/1992</year><source>The Logic of Scientific Discovery</source><publisher-loc>New York</publisher-loc><publisher-name>Routledge</publisher-name></element-citation></ref><ref id="bib38"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Prinz</surname><given-names>F</given-names></name><name><surname>Schlange</surname><given-names>T</given-names></name><name><surname>Asadullah</surname><given-names>K</given-names></name></person-group><year>2011</year><article-title>Believe it or not: how much can we rely on published data on potential drug targets?</article-title><source>Nature Reviews Drug Discovery</source><volume>10</volume><fpage>712</fpage><pub-id pub-id-type="doi">10.1038/nrd3439-c1</pub-id></element-citation></ref><ref id="bib39"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Pusztai</surname><given-names>L</given-names></name><name><surname>Hatzis</surname><given-names>C</given-names></name><name><surname>Andre</surname><given-names>F</given-names></name></person-group><year>2013</year><article-title>Reproducibility of research and preclinical validation: problems and solutions</article-title><source>Nature Reviews Clinical Oncology</source><volume>10</volume><fpage>720</fpage><lpage>724</lpage><pub-id pub-id-type="doi">10.1038/nrclinonc.2013.171</pub-id></element-citation></ref><ref id="bib40"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Rosenthal</surname><given-names>R</given-names></name><name><surname>Fode</surname><given-names>KL</given-names></name></person-group><year>1963</year><article-title>The effect of experimenter bias on the performance of the albino rat</article-title><source>Behavioral Science</source><volume>8</volume><fpage>183</fpage><lpage>189</lpage><pub-id pub-id-type="doi">10.1002/bs.3830080302</pub-id></element-citation></ref><ref id="bib41"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Schmidt</surname><given-names>S</given-names></name></person-group><year>2009</year><article-title>Shall we really do it again? The powerful concept of replication is neglected in the social sciences</article-title><source>Review of General Psychology</source><volume>13</volume><fpage>90</fpage><lpage>100</lpage><pub-id pub-id-type="doi">10.1037/a0015108</pub-id></element-citation></ref><ref id="bib42"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Simmons</surname><given-names>J</given-names></name><name><surname>Nelson</surname><given-names>L</given-names></name><name><surname>Simonsohn</surname><given-names>U</given-names></name></person-group><year>2011</year><article-title>False-positive psychology: Undisclosed flexibility in data collection and analysis allows presenting anything as &#x201c;significant&#x201d;</article-title><source>Psychological Science</source><volume>22</volume><fpage>1359</fpage><lpage>1366</lpage><pub-id pub-id-type="doi">10.1177/0956797611417632</pub-id></element-citation></ref><ref id="bib43"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Sterling</surname><given-names>TD</given-names></name></person-group><year>1959</year><article-title>Publication decisions and their possible effects on inferences drawn from tests of significance&#x2014;or vice versa</article-title><source>Journal of the American Statistical Association</source><volume>54</volume><fpage>30</fpage><lpage>34</lpage></element-citation></ref><ref id="bib44"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Steward</surname><given-names>O</given-names></name><name><surname>Popovich</surname><given-names>PG</given-names></name><name><surname>Dietrich</surname><given-names>WD</given-names></name><name><surname>Kleitman</surname><given-names>N</given-names></name></person-group><year>2012</year><article-title>Replication and reproducibility in spinal cord injury research</article-title><source>Experimental Neurology</source><volume>233</volume><fpage>597</fpage><lpage>605</lpage><pub-id pub-id-type="doi">10.1016/j.expneurol.2011.06.017</pub-id></element-citation></ref><ref id="bib45"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Valentine</surname><given-names>JC</given-names></name><name><surname>Biglan</surname><given-names>A</given-names></name><name><surname>Boruch</surname><given-names>RF</given-names></name><name><surname>Castro</surname><given-names>FG</given-names></name><name><surname>Collins</surname><given-names>LM</given-names></name><name><surname>Flay</surname><given-names>BR</given-names></name><name><surname>Kellam</surname><given-names>S</given-names></name><name><surname>Moscicki</surname><given-names>EK</given-names></name><name><surname>Schinke</surname><given-names>SP</given-names></name></person-group><year>2011</year><article-title>Replication in prevention science</article-title><source>Prevention Science</source><volume>12</volume><fpage>103</fpage><lpage>117</lpage><pub-id pub-id-type="doi">10.1007/s11121-011-0217-6</pub-id></element-citation></ref><ref id="bib46"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Vasilevsky</surname><given-names>NA</given-names></name><name><surname>Brush</surname><given-names>MH</given-names></name><name><surname>Paddock</surname><given-names>H</given-names></name><name><surname>Ponting</surname><given-names>L</given-names></name><name><surname>Tripathy</surname><given-names>SJ</given-names></name><name><surname>LaRocca</surname><given-names>GM</given-names></name><name><surname>Haendel</surname><given-names>MA</given-names></name></person-group><year>2013</year><article-title>On the reproducibility of science: unique identification of research resources in the biomedical literature</article-title><source>PeerJ</source><volume>1</volume><fpage>e148</fpage><pub-id pub-id-type="doi">10.7717/peerj.148</pub-id></element-citation></ref><ref id="bib47"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Vines</surname><given-names>TH</given-names></name><name><surname>Albert</surname><given-names>AY</given-names></name><name><surname>Andrew</surname><given-names>RL</given-names></name><name><surname>Debarre</surname><given-names>F</given-names></name><name><surname>Bock</surname><given-names>DG</given-names></name><name><surname>Franklin</surname><given-names>MT</given-names></name><name><surname>Gilbert</surname><given-names>KJ</given-names></name><name><surname>Moore</surname><given-names>JS</given-names></name><name><surname>Renaut</surname><given-names>S</given-names></name><name><surname>Rennison</surname><given-names>DJ</given-names></name></person-group><year>2014</year><article-title>The availability of research data declines rapidly with article age</article-title><source>Current Biology</source><volume>24</volume><fpage>94</fpage><lpage>97</lpage><pub-id pub-id-type="doi">10.1016/j.cub.2013.11.014</pub-id></element-citation></ref><ref id="bib48"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Wadmann</surname><given-names>M</given-names></name></person-group><year>2013</year><article-title>NIH mulls rules for validating key results</article-title><source>Nature</source><volume>500</volume><fpage>14</fpage><lpage>16</lpage><pub-id pub-id-type="doi">10.1038/500014a</pub-id></element-citation></ref><ref id="bib49"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Wagenmakers</surname><given-names>EJ</given-names></name><name><surname>Wetzels</surname><given-names>R</given-names></name><name><surname>Borsboom</surname><given-names>D</given-names></name><name><surname>van der Maas</surname><given-names>HL</given-names></name><name><surname>Kievit</surname><given-names>RA</given-names></name></person-group><year>2012</year><article-title>An agenda for purely confirmatory research</article-title><source>Perspectives on Psychological Science</source><volume>7</volume><fpage>632</fpage><lpage>638</lpage><pub-id pub-id-type="doi">10.1177/1745691612463078</pub-id></element-citation></ref><ref id="bib50"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Wicherts</surname><given-names>JM</given-names></name><name><surname>Borsboom</surname><given-names>D</given-names></name><name><surname>Kats</surname><given-names>J</given-names></name><name><surname>Molenaar</surname><given-names>D</given-names></name></person-group><year>2006</year><article-title>The poor availability of psychological research data for reanalysis</article-title><source>The American Psychologist</source><volume>61</volume><fpage>726</fpage><lpage>728</lpage><pub-id pub-id-type="doi">10.1037/0003-066X.61.7.726</pub-id></element-citation></ref></ref-list></back><sub-article article-type="article-commentary" id="SA1"><front-stub><article-id pub-id-type="doi">10.7554/eLife.04333.003</article-id><title-group><article-title>Decision letter</article-title></title-group></front-stub><body><boxed-text><p>eLife posts the editorial decision letter and author response on a selection of the published articles (subject to the approval of the authors). An edited version of the letter sent to the authors after peer review is shown, indicating the substantive concerns or comments; minor concerns are not usually shown. Reviewers have the opportunity to discuss the decision before the letter is sent (see <ext-link ext-link-type="uri" xlink:href="http://elifesciences.org/review-process">review process</ext-link>). Similarly, the author response typically shows only responses to the major concerns raised by the reviewers.</p></boxed-text><p>Thank you for sending your work entitled &#x201c;An open investigation of the reproducibility of cancer biology research&#x201d; for consideration as a Feature article at <italic>eLife</italic>. Your article has been favorably evaluated by Sean Morrison (Senior editor), 3 reviewers from our Senior editors and Board of Reviewing Editors (including Charles Sawyers and Joan Massague), and our Features editor.</p><p>The Senior editor and the reviewers discussed their comments before we reached this decision, and the Senior editor has assembled the following comments to help you prepare a revised submission.</p><p>The reviewers were supportive of this project and believe that this introduction to the reproducibility project will be helpful for the readership of <italic>eLife</italic>. We would ask the authors to consider the following points, and to make minor revisions to the text of the article to address these issues.</p><p>1) Why the cancer field was chosen for the replication project should be made clear at the outset. Is this field more prone to publication of data that are not reproducible? One assumes that this choice was made because of the recent pharma reports in which they claimed that they could only replicate &#x223c;20% of a selected set of papers that reported new cancer drug targets. As Errington et al. point out, the data on which the pharma claims were based were not reported and it is therefore hard to evaluate them, and, as others have said, pharma is increasingly relying on NIH-funded academic labs to do the basic research to uncover new drug targets, instead of carrying out target discovery in their own groups, when they would not have this issue!</p><p>2) How many of the 50 selected cancer papers have in effect been &#x201c;replicated&#x201d; by subsequent publications on the same topic? Would this analysis require a more sophisticated analysis by an expert in the field? Of course, it is also important to note that &#x201c;me too&#x201d; studies commonly claim to replicate high impact observations as a way of publishing follow-on observations. Therefore, skepticism is warranted about claims of replication. Nonetheless, it would be worthwhile to indicate what fraction of the 50 papers have ostensibly been &#x201c;replicated&#x201d; in studies that claimed to directly test a major conclusion. We would appreciate the authors&#x2019; comments on whether it would be possible to include a table showing these data. Feel free to discuss with Sean Morrison if that would be helpful.</p><p>3) It is true that exact replication is rarely carried out, but it is unfair to say that innovation is required for publication. Perhaps, in the so-called high profile journals innovation is a criterion, but for most journals in order to be published papers have to report an advance in our understanding of a subject but do not have to be innovative in the normal sense of the word.</p><p>4) It would be worth pointing out that in most cases US federal agencies will not award grants for basic research proposals whose goal is simply to replicate, and therefore it would be hard to fund such work even if one wanted to do so.</p><p>5) The discussion about &#x201c;power&#x201d; is confusing (i.e. 21% power to detect a positive result). Perhaps the authors can provide an example (real or hypothetical) to make this point. Currently it is too abstract for most readers.</p><p>6) How many of the citations that were tallied in selecting the 50 highly cited papers for replication were in primary research articles as opposed to reviews? Arguably, reviews should have been discounted, and this might have generated a different set of &#x201c;high impact&#x201d; papers to replicate.</p><p>7) In the spirit of openness, please include a link to a spreadsheet or other file that includes the 50 papers and their citation rates and altmetric scores.</p><p>8) With regard to the discussion about the need for power factors, molecular biology experiments (e.g. sequencing, proteomics, pulldowns) often do not have a real power factor or quantification, and it is not clear how the validity of such data will be evaluated (see next point).</p><p>9) The one situation where results should have been replicated is in the lab that published the initial report, and there should be some mechanism for asking the labs how many times a particular experiment was repeated by different lab members. In some papers, one sees statements in the figure legends indicating how many times an experiment was done with similar results (sometimes in response to a reviewer), and in figures where statistical analysis is used this is usually accompanied by an indication of whether the replicates were obtained within the same experiment or between experiments.</p><p>10) While it is true that CROs and core services may not have a bias regarding what results to expect, they can or will know what was claimed in the paper describing the experiments they are replicating, which could introduce a hidden bias. Moreover, there is a concern about whether core facilities or CROs necessarily have the expertise to reproduce claims based on innovative techniques, potentially techniques developed in the laboratories that published the original claims. This concern does not necessarily undermine the overall effort, but is worth explicitly addressing.</p><p>11) The rationale for selecting the papers is well described and defensible, but it may be worth considering the fact that there are always some highly visible papers published in every field where others have trouble reproducing the work but they never publish the failure to replicate. Perhaps these papers will be chosen by the current method but it may be useful to survey a field for papers that investigators would like to see replicated. I suspect there would be a few that rise up to the top and have great impact on the field if not reproduced. That does not need to be done in the context of the current article, but perhaps worth noting that different approaches could be used in future to identify the controversial articles containing ideas that could be blocking the development of fields.</p></body></sub-article><sub-article article-type="reply" id="SA2"><front-stub><article-id pub-id-type="doi">10.7554/eLife.04333.004</article-id><title-group><article-title>Author response</article-title></title-group></front-stub><body><p><italic>1) Why the cancer field was chosen for the replication project should be made clear at the outset. Is this field more prone to publication of data that are not reproducible? One assumes that this choice was made because of the recent pharma reports in which they claimed that they could only replicate &#x223c;20% of a selected set of papers that reported new cancer drug targets. As Errington et al. point out, the data on which the pharma claims were based were not reported and it is therefore hard to evaluate them, and, as others have said, pharma is increasingly relying on NIH-funded academic labs to do the basic research to uncover new drug targets, instead of carrying out target discovery in their own groups</italic>, <italic>when they would not have this issue!</italic></p><p>We added the following to clarify why we selected cancer biology for investigation: &#x201c;We selected cancer biology as the area of focus because of the Bayer and Amgen reports, and because of the direct importance of efficient progress in this research discipline for the treatment of disease.&#x201d;</p><p><italic>2) How many of the 50 selected cancer papers have in effect been &#x201c;replicated&#x201d; by subsequent publications on the same topic? Would this analysis require a more sophisticated analysis by an expert in the field? Of course, it is also important to note that &#x201c;me too&#x201d; studies commonly claim to replicate high impact observations as a way of publishing follow-on observations. Therefore, skepticism is warranted about claims of replication. Nonetheless, it would be worthwhile to indicate what fraction of the 50 papers have ostensibly been &#x201c;replicated&#x201d; in studies that claimed to directly test a major conclusion. We would appreciate the authors&#x2019; comments on whether it would be possible to include a table showing these data. Feel free to discuss with Sean Morrison if that would be helpful</italic>.</p><p>We agree that this is an important question. For each of the replication papers, preparation of the Registered Report proposal includes a review of the literature for existing replications and the impact of the original findings on further research. This review is time-intensive and will not be completed until all of the Registered Reports have been prepared and submitted to <italic>eLife</italic>. As such, the suggested table will not be available until the final report of the Reproducibility Project: Cancer Biology in which we will summarize the aggregate results across all the studies. We will include the suggested table in that final report. We have added the following sentence: &#x201c;The replication team will also conduct a literature review for evidence of existing replications although, as noted above, direct replication is likely to be rare. Existing published evidence for replication might indicate the likelihood of reproducing the original results.&#x201d;</p><p><italic>3) It is true that exact replication is rarely carried out, but it is unfair to say that innovation is required for publication. Perhaps, in the so-called high profile journals innovation is a criterion, but for most journals in order to be published papers have to report an advance in our understanding of a subject but do not have to be innovative in the normal sense of the word</italic>.</p><p>We modified the text to address this point. It now reads, &#x201c;Incentives for scientific achievement prioritize innovation over replication (<xref ref-type="bibr" rid="bib1">Alberts et al., 2014</xref>; <xref ref-type="bibr" rid="bib33">Nosek, et al., 2012</xref>). Peer review tends to favor manuscripts that contain new findings over those that improve our understanding of a previously published finding. Moreover, careers are made by producing exciting new results at the frontiers of knowledge, not by verifying prior discoveries.&#x201d;</p><p><italic>4) It would be worth pointing out that in most cases US federal agencies will not award grants for basic research proposals whose goal is simply to replicate, and therefore it would be hard to fund such work even if one wanted to do so</italic>.</p><p>Agreed. That has been added with the following: &#x201c;These challenges are compounded by the lack of funding support available from agencies and foundations to support replication research.&#x201d;</p><p><italic>5) The discussion about &#x201c;power&#x201d; is confusing (i.e. 21% power to detect a positive result). Perhaps the authors can provide an example (real or hypothetical) to make this point. Currently it is too abstract for most readers</italic>.</p><p>We added explanation of the meaning of power and its implications for understanding the positive result rate. The text now reads: &#x201c;In neuroscience, for example, Button et al. observed the median power of studies to be 21% (<xref ref-type="bibr" rid="bib6">Button et al., 2013</xref>), which means that assuming the finding being investigated is true and accurately estimated, then only 21 of every 100 studies investigating that effect would detect statistically significant evidence for the effect. [&#x2026;] This discrepancy between observed power and observed positive results is not statistically possible. Instead, it suggests systematic exclusion of negative results (<xref ref-type="bibr" rid="bib15">Greenwald, 1975</xref>) and possibly the exaggeration of positive results by employing flexibility in analytic and reporting practices that inflate the likelihood of false positives (<xref ref-type="bibr" rid="bib42">Simmons et al., 2011</xref>).&#x201d;</p><p><italic>6) How many of the citations that were tallied in selecting the 50 highly cited papers for replication were in primary research articles as opposed to reviews? Arguably, reviews should have been discounted, and this might have generated a different set of &#x201c;high impact&#x201d; papers to replicate</italic>.</p><p>We used total citation count from the various sources. Thus, some of the citing articles could be reviews. It would be interesting to know how much the list would differ if review citations were excluded, but with present search technologies it would be very laborious to assess that at the present scale. We clarify the citation counting process, noting: &#x201c;Citations were counted from all sources, which include primary research articles and reviews.&#x201d;</p><p><italic>7) In the spirit of openness, please include a link to a spreadsheet or other file that includes the 50 papers and their citation rates and altmetric scores</italic>.</p><p>Yes, that spreadsheet is available on the OSF project page (<ext-link ext-link-type="uri" xlink:href="http://osf.io/e81xl/studies">http://osf.io/e81xl/studies</ext-link>), and we have added a link to the file in the manuscript: &#x201c;Details on the selection process and a list of the selected and excluded papers are available at the <ext-link ext-link-type="uri" xlink:href="http://osf.io/e81xl/wiki/studies">Open Science Framework</ext-link>.&#x201d;</p><p><italic>8) With regard to the discussion about the need for power factors, molecular biology experiments (e.g. sequencing, proteomics, pulldowns) often do not have a real power factor or quantification, and it is not clear how the validity of such data will be evaluated (see next point)</italic>.</p><p>We agree that this is a point worth clarifying. Sequencing and proteomics studies and experiments are excluded from this investigation because of the challenges identified by the reviewers. We added a note on this: &#x201c;Also, articles reporting sequencing results, such as publications from The Cancer Genome Atlas project, were excluded. However, if sequencing or proteomic experiments were only part of an article, the other experiments in those papers could still be eligible.&#x201d;</p><p>It is true that pulldowns are often not quantified and usually evaluated as present/not present. However, they can be quantified and usually generate very large effect sizes. We will address pulldown studies that do not have inferential tests or effect sizes in the original reports by following the approach described in the text.</p><p><italic>9) The one situation where results should have been replicated is in the lab that published the initial report, and there should be some mechanism for asking the labs how many times a particular experiment was repeated by different lab members. In some papers, one sees statements in the figure legends indicating how many times an experiment was done with similar results (sometimes in response to a reviewer), and in figures where statistical analysis is used this is usually accompanied by an indication of whether the replicates were obtained within the same experiment or between experiments</italic>.</p><p>Agreed. We have added a note that we will investigate whether there are additional replications available that are not noted in the original article: &#x201c;In these cases, we will inquire with the original authors if there are additional unpublished replications, if it is not already stated in the article, and if any details are available about the results.&#x201d;</p><p><italic>10) While it is true that CROs and core services may not have a bias regarding what results to expect, they can or will know what was claimed in the paper describing the experiments they are replicating, which could introduce a hidden bias. Moreover, there is a concern about whether core facilities or CROs necessarily have the expertise to reproduce claims based on innovative techniques, potentially techniques developed in the laboratories that published the original claims. This concern does not necessarily undermine the overall effort, but is worth explicitly addressing</italic>.</p><p>We agree that our original statement was too strong. For the first part of this comment, we revised our statement to the following: &#x201c;An advantage of these labs &#x2013; commercial contract research organizations (CROs) and core facilities &#x2013; is that they are less likely to be biased for or against replicating the effect. This may reduce the effect of experimenter expectations on observed results (<xref ref-type="bibr" rid="bib40">Rosenthal and Fode, 1963</xref>). However, it does not necessarily eliminate expectancy effects as the replicating researchers are aware of the original findings, and they may have expectations about whether the same result is likely to be obtained or not.&#x201d;</p><p>For the second part of this comment, we added the following:</p><p>&#x201c;Articles were removed if they were clinical trials, case studies, reviews, or if they required specialized samples, techniques, or equipment that would be difficult or impossible to obtain.&#x201d;</p><p>Also: &#x201c;Next, a key part of matching experiments with laboratories is to identify labs with the appropriate expertise to maximize research quality. This is particularly important with new and innovative techniques, though most techniques called for in the selected experiments are standard techniques for which expertise is widely available. As experiments are matched to labs, it is possible that no appropriate service provider can be identified. If appropriate expertise is not available, then the finding or paper will be excluded from the project.&#x201d;</p><p><italic>11) The rationale for selecting the papers is well described and defensible, but it may be worth considering the fact that there are always some highly visible papers published in every field where others have trouble reproducing the work but they never publish the failure to replicate. Perhaps these papers will be chosen by the current method but it may be useful to survey a field for papers that investigators would like to see replicated. I suspect there would be a few that rise up to the top and have great impact on the field if not reproduced. That does not need to be done in the context of the current article, but perhaps worth noting that different approaches could be used in future to identify the controversial articles containing ideas that could be blocking the development of fields</italic>.</p><p>We agree. Our selection method is one of many reasonable approaches. We added a note of possible alternatives for future research with the following comment: &#x201c;There are a variety of alternative sampling strategies that could be pursued in parallel efforts such as community nomination of findings that are important to replicate, request for authors to have their published findings replicated (e.g., the <ext-link ext-link-type="uri" xlink:href="http://validation.scienceexchange.com/">Reproducibility Initiative project</ext-link>), or selection of a sample from a particular journal or on a specific topic for focused investigation.&#x201d;</p></body></sub-article></article>