What type of Robots.txt file allow all bot to visit my site, my blog and make index it?

citadelsystems Level 1

October 31, 2012, 11:10 pm

Write the reason you're deleting this FAQ

What type of Robots.txt file allow all bot to visit my site, my blog and make index it?

Hello everyone...
I want to know that what extension helps to allow all bot to index our sites and crawl them.
thanks

Comments

Please login or sign up to leave a comment

Join

rannt

Put this into your robot.txt file:

User-agent: *
Disallow:

Thank you
Gašper

14 years ago

Are you sure you want to delete this post?

From what I understand with the robot crawlers of search engines, they take on regular sites with no problem. Regular site means you have not robot.tx in your site because that file is an instruction to the robot crawlers on what to do. If there is no robot.txt file then that means there is no instruction so the robot crawlers will do the default which is to crawl on your web pages.

7 years ago

Are you sure you want to delete this post?

SEOClerks

What type of Robots.txt file allow all bot to visit my site, my blog and make index it?

What type of Robots.txt file allow all bot to visit my site, my blog and make index it?

Comments

Please login or sign up to leave a comment

rannt

Corzhens

Invalid SQL

Expensive SQL

Suspicious SQL

Match	Count	SQL	Script
SELECT * FROM members_ledger WHERE ip = ? AND added>=unix_timestamp(NOW())-86400	1	SELECT querystring, added FROM members_ledger WHERE ip = ? AND added>=unix_timestamp(NOW())-86400	/opt/clerks-staging/docroot/include/functions/includes/security.php 398 fetchMemberLedger() include_once()
SELECT * FROM `questions` as q JOIN categories_faq as c ON q.catid=c.CATID WHERE (q.status=1 OR (q.status=2 AND userid='') ) AND q.quesid='1709'	1	SELECT q.*, c.seo as CatSEO, c.name as CatName, c.parentid FROM `questions` as q JOIN categories_faq as c ON q.catid=c.CATID WHERE (q.status=1 OR (q.status=2 AND userid='') ) AND q.quesid='1709'	/opt/clerks-staging/docroot/viewfaq.php 496 () ()
SELECT * FROM seoclerks.members WHERE USERID='9399'	1	SELECT * FROM seoclerks.members WHERE USERID='9399'	/opt/clerks-staging/docroot/include/functions/includes/member.php 445 GetAllUserDetails() ()
SELECT * FROM seoclerks.members WHERE USERID=9399	1	SELECT googleplus_profile FROM seoclerks.members WHERE USERID=9399	/opt/clerks-staging/docroot/include/functions/main.php 16914 GetGooglePlusProfileFromId() ()
SELECT * FROM ratings_faq as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=1709 LIMIT 5	1	SELECT m.username FROM ratings_faq as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=1709 LIMIT 5	/opt/clerks-staging/docroot/include/functions/main.php 17159 GetVoters() ()
SELECT * FROM answers a, seoclerks.members b WHERE a.quesid='1709' AND a.userid=b.USERID and b.status='1' AND a.status=1 ORDER BY a.combined_votes DESC, a.date_answered asc	1	SELECT a.answer, a.USERID, a.upvotes, a.downvotes, a.ansid, a.parentid, a.combined_votes, a.date_answered, b.username, b.userlevel, b.profilepicture FROM answers a, seoclerks.members b WHERE a.quesid='1709' AND a.userid=b.USERID and b.status='1' AND a.status=1 ORDER BY a.combined_votes DESC, a.date_answered asc	/opt/clerks-staging/docroot/viewfaq.php 1245 () ()
SELECT * FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=10656 LIMIT 5	1	SELECT m.username FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=10656 LIMIT 5	/opt/clerks-staging/docroot/include/functions/main.php 17159 GetVoters() ()
SELECT * FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=185436 LIMIT 5	1	SELECT m.username FROM ratings_faqanswers as r, seoclerks.members as m WHERE r.USERID=m.USERID AND r.upvote=1 AND r.PID=185436 LIMIT 5	/opt/clerks-staging/docroot/include/functions/main.php 17159 GetVoters() ()
UPDATE questions SET total_views = total_views + 1 WHERE quesid='1709'	1	UPDATE questions SET total_views = total_views + 1 WHERE quesid='1709'	/opt/clerks-staging/docroot/include/functions/main.php 1765 update_Faqviewcount() ()
SELECT * FROM questions WHERE quesid!='1709' AND status='1' AND (question like '% type Robots.txt file bot visit site, blog index it?%' OR question like '%type%' OR question like '%Robots.txt%' OR question like '%file%' OR question like '%visit%' OR question like '%site,%' OR question like '%blog%' OR question like '%index%') -- ORDER BY RAND() LIMIT 15	1	SELECT quesid, question, seo, userid FROM questions WHERE quesid!='1709' AND status='1' AND (question like '% type Robots.txt file bot visit site, blog index it?%' OR question like '%type%' OR question like '%Robots.txt%' OR question like '%file%' OR question like '%visit%' OR question like '%site,%' OR question like '%blog%' OR question like '%index%') -- ORDER BY RAND() LIMIT 15	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() ()
SELECT * FROM seoclerks.members WHERE USERID='43' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='43' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM seoclerks.members WHERE USERID='9' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='9' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM seoclerks.members WHERE USERID='632' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='632' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM seoclerks.members WHERE USERID='707' limit 1	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='707' limit 1	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_member_profilepicture()
SELECT * FROM members_ledger WHERE script='/opt/clerks-staging/docroot/viewfaq.php' AND querystring LIKE '%id=1709%' AND added>=UNIX_TIMESTAMP(NOW())-1200 GROUP BY USERID	1	SELECT USERID, username FROM members_ledger WHERE script='/opt/clerks-staging/docroot/viewfaq.php' AND querystring LIKE '%id=1709%' AND added>=UNIX_TIMESTAMP(NOW())-1200 GROUP BY USERID	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() ()
SELECT * FROM categories	1	SELECT * FROM categories	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() parseRedundantQueriesCache()
select * from categories_software order by name asc	1	select * from categories_software order by name asc	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_GetSoftwareCategories()
select * from categories_wanttobuy order by name asc	1	select * from categories_wanttobuy order by name asc	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_wantcategories()
select * from categories_wanttotrade order by name asc	1	select * from categories_wanttotrade order by name asc	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_get_tradecategories()
SELECT * FROM seoclerks.members WHERE USERID='7453.jpg'	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='7453.jpg'	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() getUserProfileImage()
SELECT * FROM seoclerks.members WHERE USERID='600657.jpg'	1	SELECT profilepicture FROM seoclerks.members WHERE USERID='600657.jpg'	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() getUserProfileImage()
select * from categories_faq order by name asc	1	select * from categories_faq order by name asc	/opt/clerks-staging/docroot/libraries/adodb5/adodb.inc.php 1899 CacheExecute() insert_GetFaqCategories()